using Lucene.Net.Analysis.Standard;
using Lucene.Net.Analysis.TokenAttributes;
using Lucene.Net.Attributes;
using Lucene.Net.Util;
using NUnit.Framework;
using System;
using System.IO;

namespace Lucene.Net.Analysis.Core
{
    /*
	 * Licensed to the Apache Software Foundation (ASF) under one or more
	 * contributor license agreements.  See the NOTICE file distributed with
	 * this work for additional information regarding copyright ownership.
	 * The ASF licenses this file to You under the Apache License, Version 2.0
	 * (the "License"); you may not use this file except in compliance with
	 * the License.  You may obtain a copy of the License at
	 *
	 *     http://www.apache.org/licenses/LICENSE-2.0
	 *
	 * Unless required by applicable law or agreed to in writing, software
	 * distributed under the License is distributed on an "AS IS" BASIS,
	 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	 * See the License for the specific language governing permissions and
	 * limitations under the License.
	 */

    public class TestAnalyzers : BaseTokenStreamTestCase
    {

        [Test]
        public virtual void TestSimple()
        {
            Analyzer a = new SimpleAnalyzer(TEST_VERSION_CURRENT);
            AssertAnalyzesTo(a, "foo bar FOO BAR", new string[] { "foo", "bar", "foo", "bar" });
            AssertAnalyzesTo(a, "foo      bar .  FOO <> BAR", new string[] { "foo", "bar", "foo", "bar" });
            AssertAnalyzesTo(a, "foo.bar.FOO.BAR", new string[] { "foo", "bar", "foo", "bar" });
            AssertAnalyzesTo(a, "U.S.A.", new string[] { "u", "s", "a" });
            AssertAnalyzesTo(a, "C++", new string[] { "c" });
            AssertAnalyzesTo(a, "B2B", new string[] { "b", "b" });
            AssertAnalyzesTo(a, "2B", new string[] { "b" });
            AssertAnalyzesTo(a, "\"QUOTED\" word", new string[] { "quoted", "word" });
        }

        [Test]
        public virtual void TestNull()
        {
            Analyzer a = new WhitespaceAnalyzer(TEST_VERSION_CURRENT);
            AssertAnalyzesTo(a, "foo bar FOO BAR", new string[] { "foo", "bar", "FOO", "BAR" });
            AssertAnalyzesTo(a, "foo      bar .  FOO <> BAR", new string[] { "foo", "bar", ".", "FOO", "<>", "BAR" });
            AssertAnalyzesTo(a, "foo.bar.FOO.BAR", new string[] { "foo.bar.FOO.BAR" });
            AssertAnalyzesTo(a, "U.S.A.", new string[] { "U.S.A." });
            AssertAnalyzesTo(a, "C++", new string[] { "C++" });
            AssertAnalyzesTo(a, "B2B", new string[] { "B2B" });
            AssertAnalyzesTo(a, "2B", new string[] { "2B" });
            AssertAnalyzesTo(a, "\"QUOTED\" word", new string[] { "\"QUOTED\"", "word" });
        }

        [Test]
        public virtual void TestStop()
        {
            Analyzer a = new StopAnalyzer(TEST_VERSION_CURRENT);
            AssertAnalyzesTo(a, "foo bar FOO BAR", new string[] { "foo", "bar", "foo", "bar" });
            AssertAnalyzesTo(a, "foo a bar such FOO THESE BAR", new string[] { "foo", "bar", "foo", "bar" });
        }
        internal virtual void VerifyPayload(TokenStream ts)
        {
            IPayloadAttribute payloadAtt = ts.GetAttribute<IPayloadAttribute>();
            ts.Reset();
            for (sbyte b = 1; ; b++)
            {
                bool hasNext = ts.IncrementToken();
                if (!hasNext)
                {
                    break;
                }
                // System.out.println("id="+System.identityHashCode(nextToken) + " " + t);
                // System.out.println("payload=" + (int)nextToken.getPayload().toByteArray()[0]);
                assertEquals(b, payloadAtt.Payload.Bytes[0]);
            }
        }

        // Make sure old style next() calls result in a new copy of payloads
        [Test]
        public virtual void TestPayloadCopy()
        {
            string s = "how now brown cow";
            TokenStream ts;
            ts = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(s));
            ts = new PayloadSetter(ts);
            VerifyPayload(ts);

            ts = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(s));
            ts = new PayloadSetter(ts);
            VerifyPayload(ts);
        }

        // LUCENE-1150: Just a compile time test, to ensure the
        // StandardAnalyzer constants remain publicly accessible
        public virtual void _TestStandardConstants()
        {
#pragma warning disable 219, 612, 618
            int x = StandardTokenizer.ALPHANUM;
            x = StandardTokenizer.APOSTROPHE;
            x = StandardTokenizer.ACRONYM;
            x = StandardTokenizer.COMPANY;
            x = StandardTokenizer.EMAIL;
            x = StandardTokenizer.HOST;
            x = StandardTokenizer.NUM;
            x = StandardTokenizer.CJ;
            string[] y = StandardTokenizer.TOKEN_TYPES;
#pragma warning restore 219, 612, 618
        }

        private class LowerCaseWhitespaceAnalyzer : Analyzer
        {

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
                return new TokenStreamComponents(tokenizer, new LowerCaseFilter(TEST_VERSION_CURRENT, tokenizer));
            }

        }

        private class UpperCaseWhitespaceAnalyzer : Analyzer
        {

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
                return new TokenStreamComponents(tokenizer, new UpperCaseFilter(TEST_VERSION_CURRENT, tokenizer));
            }

        }


        /// <summary>
        /// Test that LowercaseFilter handles entire unicode range correctly
        /// </summary>
        [Test]
        public virtual void TestLowerCaseFilter()
        {
            Analyzer a = new LowerCaseWhitespaceAnalyzer();
            // BMP
            AssertAnalyzesTo(a, "AbaCaDabA", new string[] { "abacadaba" });
            // supplementary
            AssertAnalyzesTo(a, "\ud801\udc16\ud801\udc16\ud801\udc16\ud801\udc16", new string[] { "\ud801\udc3e\ud801\udc3e\ud801\udc3e\ud801\udc3e" });
            AssertAnalyzesTo(a, "AbaCa\ud801\udc16DabA", new string[] { "abaca\ud801\udc3edaba" });
            // unpaired lead surrogate
            AssertAnalyzesTo(a, "AbaC\uD801AdaBa", new string[] { "abac\uD801adaba" });
            // unpaired trail surrogate
            AssertAnalyzesTo(a, "AbaC\uDC16AdaBa", new string[] { "abac\uDC16adaba" });
        }

        /// <summary>
        /// Test that LowercaseFilter handles entire unicode range correctly
        /// </summary>
        [Test]
        public virtual void TestUpperCaseFilter()
        {
            Analyzer a = new UpperCaseWhitespaceAnalyzer();
            // BMP
            AssertAnalyzesTo(a, "AbaCaDabA", new string[] { "ABACADABA" });
            // supplementary
            AssertAnalyzesTo(a, "\ud801\udc3e\ud801\udc3e\ud801\udc3e\ud801\udc3e", new string[] { "\ud801\udc16\ud801\udc16\ud801\udc16\ud801\udc16" });
            AssertAnalyzesTo(a, "AbaCa\ud801\udc3eDabA", new string[] { "ABACA\ud801\udc16DABA" });
            // unpaired lead surrogate
            AssertAnalyzesTo(a, "AbaC\uD801AdaBa", new string[] { "ABAC\uD801ADABA" });
            // unpaired trail surrogate
            AssertAnalyzesTo(a, "AbaC\uDC16AdaBa", new string[] { "ABAC\uDC16ADABA" });
        }


        /// <summary>
        /// Test that LowercaseFilter handles the lowercasing correctly if the term
        /// buffer has a trailing surrogate character leftover and the current term in
        /// the buffer ends with a corresponding leading surrogate.
        /// </summary>
        [Test]
        public virtual void TestLowerCaseFilterLowSurrogateLeftover()
        {
            // test if the limit of the termbuffer is correctly used with supplementary
            // chars
            WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("BogustermBogusterm\udc16"));
            LowerCaseFilter filter = new LowerCaseFilter(TEST_VERSION_CURRENT, tokenizer);
            AssertTokenStreamContents(filter, new string[] { "bogustermbogusterm\udc16" });
            filter.Reset();
            string highSurEndingUpper = "BogustermBoguster\ud801";
            string highSurEndingLower = "bogustermboguster\ud801";
            tokenizer.SetReader(new StringReader(highSurEndingUpper));
            AssertTokenStreamContents(filter, new string[] { highSurEndingLower });
            assertTrue(filter.HasAttribute<ICharTermAttribute>());
            char[] termBuffer = filter.GetAttribute<ICharTermAttribute>().Buffer;
            int length = highSurEndingLower.Length;
            assertEquals('\ud801', termBuffer[length - 1]);
        }

        [Test]
        public virtual void TestLowerCaseTokenizer()
        {
            StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
            LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, reader);
            AssertTokenStreamContents(tokenizer, new string[] { "tokenizer", "\ud801\udc44test" });
        }

        [Test]
        [Obsolete("deprecated (3.1)")]
        public virtual void TestLowerCaseTokenizerBWCompat()
        {
            StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
            LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(LuceneVersion.LUCENE_30, reader);
            AssertTokenStreamContents(tokenizer, new string[] { "tokenizer", "test" });
        }

        [Test]
        public virtual void TestWhitespaceTokenizer()
        {
            StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
            WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
            AssertTokenStreamContents(tokenizer, new string[] { "Tokenizer", "\ud801\udc1ctest" });
        }

        [Test]
        [Obsolete("deprecated (3.1)")]
        public virtual void TestWhitespaceTokenizerBWCompat()
        {
            StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
            WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(LuceneVersion.LUCENE_30, reader);
            AssertTokenStreamContents(tokenizer, new string[] { "Tokenizer", "\ud801\udc1ctest" });
        }

        /// <summary>
        /// blast some random strings through the analyzer </summary>
        [Test]
        public virtual void TestRandomStrings()
        {
            CheckRandomData(Random, new WhitespaceAnalyzer(TEST_VERSION_CURRENT), 1000 * RANDOM_MULTIPLIER);
            CheckRandomData(Random, new SimpleAnalyzer(TEST_VERSION_CURRENT), 1000 * RANDOM_MULTIPLIER);
            CheckRandomData(Random, new StopAnalyzer(TEST_VERSION_CURRENT), 1000 * RANDOM_MULTIPLIER);
        }

        /// <summary>
        /// blast some random large strings through the analyzer </summary>
        [Test, LongRunningTest]
        public virtual void TestRandomHugeStrings()
        {
            Random random = Random;
            CheckRandomData(random, new WhitespaceAnalyzer(TEST_VERSION_CURRENT), 100 * RANDOM_MULTIPLIER, 8192);
            CheckRandomData(random, new SimpleAnalyzer(TEST_VERSION_CURRENT), 100 * RANDOM_MULTIPLIER, 8192);
            CheckRandomData(random, new StopAnalyzer(TEST_VERSION_CURRENT), 100 * RANDOM_MULTIPLIER, 8192);
        }
    }

    internal sealed class PayloadSetter : TokenFilter
    {
        private void InitializeInstanceFields()
        {
            p = new BytesRef(data, 0, 1);
        }

        internal IPayloadAttribute payloadAtt;
        public PayloadSetter(TokenStream input) : base(input)
        {
            InitializeInstanceFields();
            payloadAtt = AddAttribute<IPayloadAttribute>();
        }

        internal byte[] data = new byte[1];
        internal BytesRef p;

        public override bool IncrementToken()
        {
            bool hasNext = m_input.IncrementToken();
            if (!hasNext)
            {
                return false;
            }
            payloadAtt.Payload = p; // reuse the payload / byte[]
            data[0]++;
            return true;
        }
    }
}