using Lucene.Net.Analysis;
using Lucene.Net.Analysis.TokenAttributes;
using Lucene.Net.Documents;
using Lucene.Net.Support;
using Lucene.Net.Util;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Console = Lucene.Net.Support.SystemConsole;

namespace Lucene.Net.Search.Suggest.Analyzing
{
    /*
     * Licensed to the Apache Software Foundation (ASF) under one or more
     * contributor license agreements.  See the NOTICE file distributed with
     * this work for additional information regarding copyright ownership.
     * The ASF licenses this file to You under the Apache License, Version 2.0
     * (the "License"); you may not use this file except in compliance with
     * the License.  You may obtain a copy of the License at
     *
     *     http://www.apache.org/licenses/LICENSE-2.0
     *
     * Unless required by applicable law or agreed to in writing, software
     * distributed under the License is distributed on an "AS IS" BASIS,
     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     * See the License for the specific language governing permissions and
     * limitations under the License.
     */

    public class AnalyzingSuggesterTest : LuceneTestCase
    {
        /** this is basically the WFST test ported to KeywordAnalyzer. so it acts the same */
        [Test]
        public void TestKeyword()
        {
            IEnumerable<Input> keys = Shuffle(
                new Input("foo", 50),
                new Input("bar", 10),
                new Input("barbar", 10),
                new Input("barbar", 12),
                new Input("barbara", 6),
                new Input("bar", 5),
                new Input("barbara", 1)
            );

            AnalyzingSuggester suggester = new AnalyzingSuggester(new MockAnalyzer(Random, MockTokenizer.KEYWORD, false));
            suggester.Build(new InputArrayIterator(keys));

            // top N of 2, but only foo is available
            IList<Lookup.LookupResult> results = suggester.DoLookup(TestUtil.StringToCharSequence("f", Random).ToString(), false, 2);
            assertEquals(1, results.size());
            assertEquals("foo", results.ElementAt(0).Key.toString());
            assertEquals(50, results.ElementAt(0).Value, 0.01F);

            // top N of 1 for 'bar': we return this even though
            // barbar is higher because exactFirst is enabled:
            results = suggester.DoLookup(TestUtil.StringToCharSequence("bar", Random).ToString(), false, 1);
            assertEquals(1, results.size());
            assertEquals("bar", results.ElementAt(0).Key.toString());
            assertEquals(10, results.ElementAt(0).Value, 0.01F);

            // top N Of 2 for 'b'
            results = suggester.DoLookup(TestUtil.StringToCharSequence("b", Random).ToString(), false, 2);
            assertEquals(2, results.size());
            assertEquals("barbar", results.ElementAt(0).Key.toString());
            assertEquals(12, results.ElementAt(0).Value, 0.01F);
            assertEquals("bar", results.ElementAt(1).Key.toString());
            assertEquals(10, results.ElementAt(1).Value, 0.01F);

            // top N of 3 for 'ba'
            results = suggester.DoLookup(TestUtil.StringToCharSequence("ba", Random).ToString(), false, 3);
            assertEquals(3, results.size());
            assertEquals("barbar", results.ElementAt(0).Key.toString());
            assertEquals(12, results.ElementAt(0).Value, 0.01F);
            assertEquals("bar", results.ElementAt(1).Key.toString());
            assertEquals(10, results.ElementAt(1).Value, 0.01F);
            assertEquals("barbara", results.ElementAt(2).Key.toString());
            assertEquals(6, results.ElementAt(2).Value, 0.01F);
        }

        [Test]
        public void TestKeywordWithPayloads()
        {
            IEnumerable<Input> keys = Shuffle(
                new Input("foo", 50, new BytesRef("hello")),
                new Input("bar", 10, new BytesRef("goodbye")),
                new Input("barbar", 12, new BytesRef("thank you")),
                new Input("bar", 9, new BytesRef("should be deduplicated")),
                new Input("bar", 8, new BytesRef("should also be deduplicated")),
                new Input("barbara", 6, new BytesRef("for all the fish")));

            AnalyzingSuggester suggester = new AnalyzingSuggester(new MockAnalyzer(Random, MockTokenizer.KEYWORD, false));
            suggester.Build(new InputArrayIterator(keys));
            for (int i = 0; i < 2; i++)
            {
                // top N of 2, but only foo is available
                IList<Lookup.LookupResult> results = suggester.DoLookup(TestUtil.StringToCharSequence("f", Random).ToString(), false, 2);
                assertEquals(1, results.size());
                assertEquals("foo", results.ElementAt(0).Key.toString());
                assertEquals(50, results.ElementAt(0).Value, 0.01F);
                assertEquals(new BytesRef("hello"), results.ElementAt(0).Payload);

                // top N of 1 for 'bar': we return this even though
                // barbar is higher because exactFirst is enabled:
                results = suggester.DoLookup(TestUtil.StringToCharSequence("bar", Random).ToString(), false, 1);
                assertEquals(1, results.size());
                assertEquals("bar", results.ElementAt(0).Key.toString());
                assertEquals(10, results.ElementAt(0).Value, 0.01F);
                assertEquals(new BytesRef("goodbye"), results.ElementAt(0).Payload);

                // top N Of 2 for 'b'
                results = suggester.DoLookup(TestUtil.StringToCharSequence("b", Random).ToString(), false, 2);
                assertEquals(2, results.size());
                assertEquals("barbar", results.ElementAt(0).Key.toString());
                assertEquals(12, results.ElementAt(0).Value, 0.01F);
                assertEquals(new BytesRef("thank you"), results.ElementAt(0).Payload);
                assertEquals("bar", results.ElementAt(1).Key.toString());
                assertEquals(10, results.ElementAt(1).Value, 0.01F);
                assertEquals(new BytesRef("goodbye"), results.ElementAt(1).Payload);

                // top N of 3 for 'ba'
                results = suggester.DoLookup(TestUtil.StringToCharSequence("ba", Random).ToString(), false, 3);
                assertEquals(3, results.size());
                assertEquals("barbar", results.ElementAt(0).Key.toString());
                assertEquals(12, results.ElementAt(0).Value, 0.01F);
                assertEquals(new BytesRef("thank you"), results.ElementAt(0).Payload);
                assertEquals("bar", results.ElementAt(1).Key.toString());
                assertEquals(10, results.ElementAt(1).Value, 0.01F);
                assertEquals(new BytesRef("goodbye"), results.ElementAt(1).Payload);
                assertEquals("barbara", results.ElementAt(2).Key.toString());
                assertEquals(6, results.ElementAt(2).Value, 0.01F);
                assertEquals(new BytesRef("for all the fish"), results.ElementAt(2).Payload);
            }
        }

        [Test]
        public void TestRandomRealisticKeys()
        {
            LineFileDocs lineFile = new LineFileDocs(Random);
            IDictionary<string, long> mapping = new HashMap<string, long>();
            List<Input> keys = new List<Input>();

            int howMany = AtLeast(100); // this might bring up duplicates
            for (int i = 0; i < howMany; i++)
            {
                Document nextDoc = lineFile.NextDoc();
                string title = nextDoc.GetField("title").GetStringValue();
                int randomWeight = Random.nextInt(100);
                keys.Add(new Input(title, randomWeight));
                if (!mapping.ContainsKey(title) || mapping[title] < randomWeight)
                {
                    mapping.Put(title, Convert.ToInt64(randomWeight));
                }
            }
            AnalyzingSuggester analyzingSuggester = new AnalyzingSuggester(new MockAnalyzer(Random), new MockAnalyzer(Random),
                SuggesterOptions.EXACT_FIRST | SuggesterOptions.PRESERVE_SEP, 256, -1, Random.nextBoolean());
            bool doPayloads = Random.nextBoolean();
            if (doPayloads)
            {
                List<Input> keysAndPayloads = new List<Input>();
                foreach (Input termFreq in keys)
                {
                    keysAndPayloads.Add(new Input(termFreq.term, termFreq.v, new BytesRef(termFreq.v.ToString())));
                }
                analyzingSuggester.Build(new InputArrayIterator(keysAndPayloads));
            }
            else
            {
                analyzingSuggester.Build(new InputArrayIterator(keys));
            }

            foreach (Input termFreq in keys)
            {
                IList<Lookup.LookupResult> lookup = analyzingSuggester.DoLookup(termFreq.term.Utf8ToString(), false, keys.size());
                foreach (Lookup.LookupResult lookupResult in lookup)
                {
                    assertEquals(mapping[lookupResult.Key], lookupResult.Value);
                    if (doPayloads)
                    {
                        assertEquals(lookupResult.Payload.Utf8ToString(), lookupResult.Value.ToString());
                    }
                    else
                    {
                        assertNull(lookupResult.Payload);
                    }
                }
            }

            lineFile.Dispose();
        }

        // TODO: more tests
        /**
         * basic "standardanalyzer" test with stopword removal
         */
        [Test]
        public void TestStandard()
        {
            Input[] keys = new Input[] {
                new Input("the ghost of christmas past", 50),
            };

            Analyzer standard = new MockAnalyzer(Random, MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
            AnalyzingSuggester suggester = new AnalyzingSuggester(standard, standard,
                SuggesterOptions.EXACT_FIRST | SuggesterOptions.PRESERVE_SEP, 256, -1, false);

            suggester.Build(new InputArrayIterator(keys));

            IList<Lookup.LookupResult> results = suggester.DoLookup(TestUtil.StringToCharSequence("the ghost of chris", Random).ToString(), false, 1);
            assertEquals(1, results.size());
            assertEquals("the ghost of christmas past", results.ElementAt(0).Key.toString());
            assertEquals(50, results.ElementAt(0).Value, 0.01F);

            // omit the 'the' since its a stopword, its suggested anyway
            results = suggester.DoLookup(TestUtil.StringToCharSequence("ghost of chris", Random).ToString(), false, 1);
            assertEquals(1, results.size());
            assertEquals("the ghost of christmas past", results.ElementAt(0).Key.toString());
            assertEquals(50, results.ElementAt(0).Value, 0.01F);

            // omit the 'the' and 'of' since they are stopwords, its suggested anyway
            results = suggester.DoLookup(TestUtil.StringToCharSequence("ghost chris", Random).ToString(), false, 1);
            assertEquals(1, results.size());
            assertEquals("the ghost of christmas past", results.ElementAt(0).Key.toString());
            assertEquals(50, results.ElementAt(0).Value, 0.01F);
        }

        [Test]
        public void TestEmpty()
        {
            Analyzer standard = new MockAnalyzer(Random, MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
            AnalyzingSuggester suggester = new AnalyzingSuggester(standard);
            suggester.Build(new InputArrayIterator(new Input[0]));

            IList<Lookup.LookupResult> result = suggester.DoLookup("a", false, 20);
            assertTrue(!result.Any());
        }

        [Test]
        public void TestNoSeps()
        {
            Input[]
            keys = new Input[] {
                new Input("ab cd", 0),
                new Input("abcd", 1),
            };

            SuggesterOptions options = 0;

            Analyzer a = new MockAnalyzer(Random);
            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, options, 256, -1, true);
            suggester.Build(new InputArrayIterator(keys));
            // TODO: would be nice if "ab " would allow the test to
            // pass, and more generally if the analyzer can know
            // that the user's current query has ended at a word, 
            // but, analyzers don't produce SEP tokens!
            IList<Lookup.LookupResult> r = suggester.DoLookup(TestUtil.StringToCharSequence("ab c", Random).ToString(), false, 2);
            assertEquals(2, r.size());

            // With no PRESERVE_SEPS specified, "ab c" should also
            // complete to "abcd", which has higher weight so should
            // appear first:
            assertEquals("abcd", r.ElementAt(0).Key.toString());
        }

        internal class TestGraphDupsTokenStreamComponents : TokenStreamComponents
        {
            private readonly AnalyzingSuggesterTest outerInstance;
            internal int tokenStreamCounter = 0;
            internal readonly TokenStream[] tokenStreams = new TokenStream[] {
            new CannedTokenStream(new Token[] {
                AnalyzingSuggesterTest.NewToken("wifi",1,1),
                AnalyzingSuggesterTest.NewToken("hotspot",0,2),
                AnalyzingSuggesterTest.NewToken("network",1,1),
                AnalyzingSuggesterTest.NewToken("is",1,1),
                AnalyzingSuggesterTest.NewToken("slow",1,1)
              }),
            new CannedTokenStream(new Token[] {
                AnalyzingSuggesterTest.NewToken("wi",1,1),
                AnalyzingSuggesterTest.NewToken("hotspot",0,3),
                AnalyzingSuggesterTest.NewToken("fi",1,1),
                AnalyzingSuggesterTest.NewToken("network",1,1),
                AnalyzingSuggesterTest.NewToken("is",1,1),
                AnalyzingSuggesterTest.NewToken("fast",1,1)

              }),
            new CannedTokenStream(new Token[] {
                AnalyzingSuggesterTest.NewToken("wifi",1,1),
                AnalyzingSuggesterTest.NewToken("hotspot",0,2),
                AnalyzingSuggesterTest.NewToken("network",1,1)
              }),
            };

            public TestGraphDupsTokenStreamComponents(AnalyzingSuggesterTest outerInstance, Tokenizer tokenizer)
                : base(tokenizer)
            {
                this.outerInstance = outerInstance;
            }

            public override TokenStream TokenStream
            {
                get
                {
                    TokenStream result = tokenStreams[tokenStreamCounter];
                    tokenStreamCounter++;
                    return result;
                }
            }

            protected override void SetReader(TextReader reader)
            {
            }
        }

        internal class TestGraphDupsAnalyzer : Analyzer
        {
            private readonly AnalyzingSuggesterTest outerInstance;
            public TestGraphDupsAnalyzer(AnalyzingSuggesterTest outerInstance)
            {
                this.outerInstance = outerInstance;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
                return new TestGraphDupsTokenStreamComponents(outerInstance, tokenizer);
            }
        }



        [Test]
        public void TestGraphDups()
        {
            Analyzer analyzer = new TestGraphDupsAnalyzer(this);

            Input[] keys = new Input[] {
                new Input("wifi network is slow", 50),
                new Input("wi fi network is fast", 10),
            };
            //AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer, AnalyzingSuggester.EXACT_FIRST, 256, -1);
            AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer);
            suggester.Build(new InputArrayIterator(keys));
            IList<Lookup.LookupResult> results = suggester.DoLookup("wifi network", false, 10);
            if (VERBOSE)
            {
                Console.WriteLine("Results: " + results);
            }
            assertEquals(2, results.size());
            assertEquals("wifi network is slow", results.ElementAt(0).Key);
            assertEquals(50, results.ElementAt(0).Value);
            assertEquals("wi fi network is fast", results.ElementAt(1).Key);
            assertEquals(10, results.ElementAt(1).Value);
        }

        internal class TestInputPathRequiredTokenStreamComponents : TokenStreamComponents
        {
            private readonly AnalyzingSuggesterTest outerInstance;
            internal int tokenStreamCounter = 0;
            internal TokenStream[] tokenStreams = new TokenStream[] {
            new CannedTokenStream(new Token[] {
                AnalyzingSuggesterTest.NewToken("ab",1,1),
                AnalyzingSuggesterTest.NewToken("ba",0,1),
                AnalyzingSuggesterTest.NewToken("xc",1,1)
              }),
            new CannedTokenStream(new Token[] {
                AnalyzingSuggesterTest.NewToken("ba",1,1),
                AnalyzingSuggesterTest.NewToken("xd",1,1)
              }),
            new CannedTokenStream(new Token[] {
                AnalyzingSuggesterTest.NewToken("ab",1,1),
                AnalyzingSuggesterTest.NewToken("ba",0,1),
                AnalyzingSuggesterTest.NewToken("x",1,1)
              })
          };

            public TestInputPathRequiredTokenStreamComponents(AnalyzingSuggesterTest outerInstance, Tokenizer tokenizer)
                : base(tokenizer)
            {
                this.outerInstance = outerInstance;
            }

            public override TokenStream TokenStream
            {
                get
                {
                    TokenStream result = tokenStreams[tokenStreamCounter];
                    tokenStreamCounter++;
                    return result;
                }
            }

            protected override void SetReader(TextReader reader)
            {
            }
        }

        internal class TestInputPathRequiredAnalyzer : Analyzer
        {
            private readonly AnalyzingSuggesterTest outerInstance;

            public TestInputPathRequiredAnalyzer(AnalyzingSuggesterTest outerInstance)
            {
                this.outerInstance = outerInstance;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
                return new TestInputPathRequiredTokenStreamComponents(outerInstance, tokenizer);
            }
        }

        [Test]
        public void TestInputPathRequired()
        {

            //  SynonymMap.Builder b = new SynonymMap.Builder(false);
            //  b.add(new CharsRef("ab"), new CharsRef("ba"), true);
            //  final SynonymMap map = b.build();

            //  The Analyzer below mimics the functionality of the SynonymAnalyzer
            //  using the above map, so that the suggest module does not need a dependency on the 
            //  synonym module 

            Analyzer analyzer = new TestInputPathRequiredAnalyzer(this);

            Input[] keys = new Input[] {
                new Input("ab xc", 50),
                new Input("ba xd", 50),
            };
            AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer);
            suggester.Build(new InputArrayIterator(keys));
            IList<Lookup.LookupResult> results = suggester.DoLookup("ab x", false, 1);
            assertTrue(results.size() == 1);
        }

        internal static Token NewToken(string term, int posInc, int posLength)
        {
            Token t = new Token(term, 0, 0);
            t.PositionIncrement = (posInc);
            t.PositionLength = (posLength);
            return t;
        }

        internal static BinaryToken NewToken(BytesRef term)
        {
            return new BinaryToken(term);
        }

        /*
        private void printTokens(final Analyzer analyzer, String input) throws IOException {
          System.out.println("Tokens for " + input);
          TokenStream ts = analyzer.tokenStream("", new StringReader(input));
          ts.reset();
          final TermToBytesRefAttribute termBytesAtt = ts.addAttribute(TermToBytesRefAttribute.class);
          final PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);
          final PositionLengthAttribute posLengthAtt = ts.addAttribute(PositionLengthAttribute.class);

          while(ts.incrementToken()) {
            termBytesAtt.fillBytesRef();
            System.out.println(String.format("%s,%s,%s", termBytesAtt.getBytesRef().utf8ToString(), posIncAtt.getPositionIncrement(), posLengthAtt.getPositionLength()));      
          }
          ts.end();
          ts.close();
        } 
        */

        internal class UsualTokenStreamComponents : TokenStreamComponents
        {
            private readonly AnalyzingSuggesterTest outerInstance;
            internal int count;

            public UsualTokenStreamComponents(AnalyzingSuggesterTest outerInstance, Tokenizer tokenizer)
                : base(tokenizer)
            {
                this.outerInstance = outerInstance;
            }

            public override TokenStream TokenStream
            {
                get
                {
                    // 4th time we are called, return tokens a b,
                    // else just a:
                    if (count++ != 3)
                    {
                        return new CannedTokenStream(new Token[] {
                            NewToken("a", 1, 1),
                        });
                    }
                    else
                    {
                        // After that "a b":
                        return new CannedTokenStream(new Token[] {
                            NewToken("a", 1, 1),
                            NewToken("b", 1, 1),
                        });
                    }
                }
            }

            protected override void SetReader(TextReader reader)
            {
            }
        }
        internal class UsualAnalyzer : Analyzer
        {
            private readonly AnalyzingSuggesterTest outerInstance;
            public UsualAnalyzer(AnalyzingSuggesterTest outerInstance)
            {
                this.outerInstance = outerInstance;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
                return new UsualTokenStreamComponents(outerInstance, tokenizer);
            }
        }

        private Analyzer GetUnusualAnalyzer()
        {
            return new UsualAnalyzer(this);
        }

        [Test]
        public void TestExactFirst()
        {

            Analyzer a = GetUnusualAnalyzer();
            SuggesterOptions options = SuggesterOptions.EXACT_FIRST | SuggesterOptions.PRESERVE_SEP;
            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, options, 256, -1, true);
            suggester.Build(new InputArrayIterator(new Input[] {
                new Input("x y", 1),
                new Input("x y z", 3),
                new Input("x", 2),
                new Input("z z z", 20),
            }));

            //System.out.println("ALL: " + suggester.lookup("x y", false, 6));

            for (int topN = 1; topN < 6; topN++)
            {
                IList<Lookup.LookupResult> results = suggester.DoLookup("x y", false, topN);
                //System.out.println("topN=" + topN + " " + results);

                assertEquals(Math.Min(topN, 4), results.size());

                assertEquals("x y", results.ElementAt(0).Key);
                assertEquals(1, results.ElementAt(0).Value);

                if (topN > 1)
                {
                    assertEquals("z z z", results.ElementAt(1).Key);
                    assertEquals(20, results.ElementAt(1).Value);

                    if (topN > 2)
                    {
                        assertEquals("x y z", results.ElementAt(2).Key);
                        assertEquals(3, results.ElementAt(2).Value);

                        if (topN > 3)
                        {
                            assertEquals("x", results.ElementAt(3).Key);
                            assertEquals(2, results.ElementAt(3).Value);
                        }
                    }
                }
            }
        }

        [Test]
        public void TestNonExactFirst()
        {

            Analyzer a = GetUnusualAnalyzer();
            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, SuggesterOptions.PRESERVE_SEP, 256, -1, true);

            suggester.Build(new InputArrayIterator(new Input[] {
                new Input("x y", 1),
                new Input("x y z", 3),
                new Input("x", 2),
                new Input("z z z", 20),
            }));

            for (int topN = 1; topN < 6; topN++)
            {
                IList<Lookup.LookupResult> results = suggester.DoLookup("p", false, topN);

                assertEquals(Math.Min(topN, 4), results.size());

                assertEquals("z z z", results.ElementAt(0).Key);
                assertEquals(20, results.ElementAt(0).Value);

                if (topN > 1)
                {
                    assertEquals("x y z", results.ElementAt(1).Key);
                    assertEquals(3, results.ElementAt(1).Value);

                    if (topN > 2)
                    {
                        assertEquals("x", results.ElementAt(2).Key);
                        assertEquals(2, results.ElementAt(2).Value);

                        if (topN > 3)
                        {
                            assertEquals("x y", results.ElementAt(3).Key);
                            assertEquals(1, results.ElementAt(3).Value);
                        }
                    }
                }
            }
        }

        // Holds surface form separately:
        internal class TermFreq2 : IComparable<TermFreq2>
        {
            public readonly string surfaceForm;
            public readonly string analyzedForm;
            public readonly long weight;
            public readonly BytesRef payload;

            public TermFreq2(string surfaceForm, string analyzedForm, long weight, BytesRef payload)
            {
                this.surfaceForm = surfaceForm;
                this.analyzedForm = analyzedForm;
                this.weight = weight;
                this.payload = payload;
            }

            public int CompareTo(TermFreq2 other)
            {
                int cmp = analyzedForm.CompareToOrdinal(other.analyzedForm);
                if (cmp != 0)
                {
                    return cmp;
                }
                else if (weight > other.weight)
                {
                    return -1;
                }
                else if (weight < other.weight)
                {
                    return 1;
                }
                else
                {
                    Debug.Assert(false);
                    return 0;
                }
            }

            public override string ToString()
            {
                return surfaceForm + "/" + weight;
            }
        }

        internal static bool IsStopChar(char ch, int numStopChars)
        {
            //System.out.println("IS? " + ch + ": " + (ch - 'a') + ": " + ((ch - 'a') < numStopChars));
            return (ch - 'a') < numStopChars;
        }

        // Like StopFilter:
        internal sealed class TokenEater : TokenFilter
        {
            private readonly IPositionIncrementAttribute posIncrAtt;
            private readonly ICharTermAttribute termAtt;
            private readonly int numStopChars;
            private readonly bool preserveHoles;
            private bool first;

            public TokenEater(bool preserveHoles, TokenStream @in, int numStopChars)
                    : base(@in)
            {
                this.preserveHoles = preserveHoles;
                this.numStopChars = numStopChars;
                this.posIncrAtt = AddAttribute<IPositionIncrementAttribute>();
                this.termAtt = AddAttribute<ICharTermAttribute>();
            }

            public override void Reset()
            {
                base.Reset();
                first = true;
            }

            public override bool IncrementToken()
            {
                int skippedPositions = 0;
                while (m_input.IncrementToken())
                {
                    if (termAtt.Length != 1 || !IsStopChar(termAtt.Buffer[0], numStopChars))
                    {
                        int posInc = posIncrAtt.PositionIncrement + skippedPositions;
                        if (first)
                        {
                            if (posInc == 0)
                            {
                                // first token having posinc=0 is illegal.
                                posInc = 1;
                            }
                            first = false;
                        }
                        posIncrAtt.PositionIncrement = (posInc);
                        //System.out.println("RETURN term=" + termAtt + " numStopChars=" + numStopChars);
                        return true;
                    }
                    if (preserveHoles)
                    {
                        skippedPositions += posIncrAtt.PositionIncrement;
                    }
                }

                return false;
            }
        }

        internal class MockTokenEatingAnalyzer : Analyzer
        {
            private int numStopChars;
            private bool preserveHoles;

            private readonly MockBytesAttributeFactory factory = new MockBytesAttributeFactory();

            public MockTokenEatingAnalyzer(int numStopChars, bool preserveHoles)
            {
                this.preserveHoles = preserveHoles;
                this.numStopChars = numStopChars;
            }

            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                MockTokenizer tokenizer = new MockTokenizer(factory, reader, MockTokenizer.WHITESPACE, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
                tokenizer.EnableChecks = (true);
                TokenStream next;
                if (numStopChars != 0)
                {
                    next = new TokenEater(preserveHoles, tokenizer, numStopChars);
                }
                else
                {
                    next = tokenizer;
                }
                return new TokenStreamComponents(tokenizer, next);
            }
        }

        private static char SEP = '\u001F';

        internal class TestRandomComparer : IComparer<TermFreq2>
        {
            public int Compare(TermFreq2 left, TermFreq2 right)
            {
                int cmp = ((float)right.weight).CompareTo((float)left.weight);
                if (cmp == 0)
                {
                    return left.analyzedForm.CompareToOrdinal(right.analyzedForm);
                }
                else
                {
                    return cmp;
                }
            }
        }

        [Test]
        public void TestRandom()
        {

            int numQueries = AtLeast(1000);

            List<TermFreq2> slowCompletor = new List<TermFreq2>();
            ISet<string> allPrefixes = new SortedSet<string>(StringComparer.Ordinal); //new TreeSet<>();
            ISet<string> seen = new HashSet<string>();

            bool doPayloads = Random.nextBoolean();

            Input[] keys = null;
            Input[] payloadKeys = null;
            if (doPayloads)
            {
                payloadKeys = new Input[numQueries];
            }
            else
            {
                keys = new Input[numQueries];
            }

            bool preserveSep = Random.nextBoolean();

            int numStopChars = Random.nextInt(10);
            bool preserveHoles = Random.nextBoolean();

            if (VERBOSE)
            {
                Console.WriteLine("TEST: " + numQueries + " words; preserveSep=" + preserveSep + " numStopChars=" + numStopChars + " preserveHoles=" + preserveHoles);
            }

            for (int i = 0; i < numQueries; i++)
            {
                int numTokens = TestUtil.NextInt32(Random, 1, 4);
                string key;
                string analyzedKey;
                while (true)
                {
                    key = "";
                    analyzedKey = "";
                    bool lastRemoved = false;
                    for (int token = 0; token < numTokens; token++)
                    {
                        string s;
                        while (true)
                        {
                            // TODO: would be nice to fix this slowCompletor/comparer to
                            // use full range, but we might lose some coverage too...
                            s = TestUtil.RandomSimpleString(Random);
                            if (s.Length > 0)
                            {
                                if (token > 0)
                                {
                                    key += " ";
                                }
                                if (preserveSep && analyzedKey.Length > 0 && analyzedKey[analyzedKey.Length - 1] != SEP)
                                {
                                    analyzedKey += SEP;
                                }
                                key += s;
                                if (s.Length == 1 && IsStopChar(s[0], numStopChars))
                                {
                                    lastRemoved = true;
                                    if (preserveSep && preserveHoles)
                                    {
                                        analyzedKey += SEP;
                                    }
                                }
                                else
                                {
                                    lastRemoved = false;
                                    analyzedKey += s;
                                }
                                break;
                            }
                        }
                    }

                    analyzedKey = Regex.Replace(analyzedKey, "(^|" + SEP + ")" + SEP + "$", "");

                    if (preserveSep && lastRemoved)
                    {
                        analyzedKey += SEP;
                    }

                    // Don't add same surface form more than once:
                    if (!seen.contains(key))
                    {
                        seen.add(key);
                        break;
                    }
                }

                for (int j = 1; j < key.Length; j++)
                {
                    allPrefixes.add(key.Substring(0, j - 0));
                }
                // we can probably do Integer.MAX_VALUE here, but why worry.
                int weight = Random.nextInt(1 << 24);
                BytesRef payload;
                if (doPayloads)
                {
                    byte[] bytes = new byte[Random.nextInt(10)];
                    Random.NextBytes(bytes);
                    payload = new BytesRef(bytes);
                    payloadKeys[i] = new Input(key, weight, payload);
                }
                else
                {
                    keys[i] = new Input(key, weight);
                    payload = null;
                }

                slowCompletor.Add(new TermFreq2(key, analyzedKey, weight, payload));
            }

            if (VERBOSE)
            {
                // Don't just sort original list, to avoid VERBOSE
                // altering the test:
                List<TermFreq2> sorted = new List<TermFreq2>(slowCompletor);
                // LUCENENET NOTE: Must use TimSort because comparer is not expecting ties
                CollectionUtil.TimSort(sorted);
                foreach (TermFreq2 ent in sorted)
                {
                    Console.WriteLine("  surface='" + ent.surfaceForm + "' analyzed='" + ent.analyzedForm + "' weight=" + ent.weight);
                }
            }

            Analyzer a = new MockTokenEatingAnalyzer(numStopChars, preserveHoles);
            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a,
                                                                  preserveSep ? SuggesterOptions.PRESERVE_SEP : 0, 256, -1, true);
            if (doPayloads)
            {
                suggester.Build(new InputArrayIterator(Shuffle(payloadKeys)));
            }
            else
            {
                suggester.Build(new InputArrayIterator(Shuffle(keys)));
            }

            foreach (string prefix in allPrefixes)
            {

                if (VERBOSE)
                {
                    Console.WriteLine("\nTEST: prefix=" + prefix);
                }

                int topN = TestUtil.NextInt32(Random, 1, 10);
                IList<Lookup.LookupResult> r = suggester.DoLookup(TestUtil.StringToCharSequence(prefix, Random).ToString(), false, topN);

                // 2. go thru whole set to find suggestions:
                List<TermFreq2> matches = new List<TermFreq2>();

                // "Analyze" the key:
                string[] tokens = prefix.Split(' ').TrimEnd();
                StringBuilder builder = new StringBuilder();
                bool lastRemoved = false;
                for (int i = 0; i < tokens.Length; i++)
                {
                    string token = tokens[i];
                    if (preserveSep && builder.Length > 0 && !builder.ToString().EndsWith("" + SEP, StringComparison.Ordinal))
                    {
                        builder.Append(SEP);
                    }

                    if (token.Length == 1 && IsStopChar(token[0], numStopChars))
                    {
                        if (preserveSep && preserveHoles)
                        {
                            builder.Append(SEP);
                        }
                        lastRemoved = true;
                    }
                    else
                    {
                        builder.Append(token);
                        lastRemoved = false;
                    }
                }

                string analyzedKey = builder.ToString();

                // Remove trailing sep/holes (TokenStream.end() does
                // not tell us any trailing holes, yet ... there is an
                // issue open for this):
                while (true)
                {
                    string s = Regex.Replace(analyzedKey, SEP + "$", "");
                    if (s.Equals(analyzedKey, StringComparison.Ordinal))
                    {
                        break;
                    }
                    analyzedKey = s;
                }

                if (analyzedKey.Length == 0)
                {
                    // Currently suggester can't suggest from the empty
                    // string!  You get no results, not all results...
                    continue;
                }

                if (preserveSep && (prefix.EndsWith(" ", StringComparison.Ordinal) || lastRemoved))
                {
                    analyzedKey += SEP;
                }

                if (VERBOSE)
                {
                    Console.WriteLine("  analyzed: " + analyzedKey);
                }

                // TODO: could be faster... but its slowCompletor for a reason
                foreach (TermFreq2 e in slowCompletor)
                {
                    if (e.analyzedForm.StartsWith(analyzedKey, StringComparison.Ordinal))
                    {
                        matches.Add(e);
                    }
                }

                assertTrue(numStopChars > 0 || matches.size() > 0);

                if (matches.size() > 1)
                {
                    matches.Sort(new TestRandomComparer());
                }

                if (matches.size() > topN)
                {
                    //matches = new List<TermFreq2>(matches.SubList(0, topN));
                    matches = matches.GetRange(0, topN);
                }

                if (VERBOSE)
                {
                    Console.WriteLine("  expected:");
                    foreach (TermFreq2 lr in matches)
                    {
                        Console.WriteLine("    key=" + lr.surfaceForm + " weight=" + lr.weight);
                    }

                    Console.WriteLine("  actual:");
                    foreach (Lookup.LookupResult lr in r)
                    {
                        Console.WriteLine("    key=" + lr.Key + " weight=" + lr.Value);
                    }
                }

                assertEquals(matches.size(), r.size());

                for (int hit = 0; hit < r.size(); hit++)
                {
                    //System.out.println("  check hit " + hit);
                    assertEquals(matches.ElementAt(hit).surfaceForm, r.ElementAt(hit).Key);
                    assertEquals(matches.ElementAt(hit).weight, r.ElementAt(hit).Value, 0f);
                    if (doPayloads)
                    {
                        assertEquals(matches.ElementAt(hit).payload, r.ElementAt(hit).Payload);
                    }
                }
            }
        }


        /// <summary>
        /// LUCENENET specific test. Added fixed inputs to help with debugging issues found in TestRandom().
        /// Not necessarily required per se, but it may come in handy again if the above test fails.
        /// </summary>
        [Test]
        public void TestFixed()
        {
            bool preserveSep = true;
            int numStopChars = 2;
            bool preserveHoles = true;

            string token11 = "foo bar foo bar";
            string token21 = "bar foo orange cat";

            string token12 = "sally sells seashells by the sea shore";
            string token22 = "peter piper picked a pack of pickled peppers";

            string query1 = "ba";
            string query2 = "pet";

            // Query 1
            Analyzer a1 = new MockTokenEatingAnalyzer(numStopChars, preserveHoles);
            AnalyzingSuggester suggester1 = new AnalyzingSuggester(a1, a1,
                preserveSep ? SuggesterOptions.PRESERVE_SEP : 0, 256, -1, true);

            suggester1.Build(new InputArrayIterator(new Input[] { new Input(token11, 123456), new Input(token21, 654321) }));

            int topN1 = 4;
            IList<Lookup.LookupResult> r1 = suggester1.DoLookup(query1, false, topN1);

            assertEquals(1, r1.size());

            assertEquals("bar foo orange cat", r1.ElementAt(0).Key);
            assertEquals(654321, r1.ElementAt(0).Value, 0f);

            // Query 2
            Analyzer a2 = new MockTokenEatingAnalyzer(numStopChars, preserveHoles);
            AnalyzingSuggester suggester2 = new AnalyzingSuggester(a2, a2,
                preserveSep ? SuggesterOptions.PRESERVE_SEP : 0, 256, -1, true);

            suggester2.Build(new InputArrayIterator(new Input[] { new Input(token12, 1234567), new Input(token22, 7654321) }));

            int topN2 = 4;
            IList<Lookup.LookupResult> r2 = suggester2.DoLookup(query2, false, topN2);

            assertEquals(1, r2.size());

            assertEquals("peter piper picked a pack of pickled peppers", r2.ElementAt(0).Key);
            assertEquals(7654321, r2.ElementAt(0).Value, 0f);
        }


        [Test]
        public void TestMaxSurfaceFormsPerAnalyzedForm()
        {
            Analyzer a = new MockAnalyzer(Random);
            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 2, -1, true);
            suggester.Build(new InputArrayIterator(Shuffle(new Input("a", 40),
                    new Input("a ", 50), new Input(" a", 60))));

            IList<Lookup.LookupResult> results = suggester.DoLookup("a", false, 5);
            assertEquals(2, results.size());
            assertEquals(" a", results.ElementAt(0).Key);
            assertEquals(60, results.ElementAt(0).Value);
            assertEquals("a ", results.ElementAt(1).Key);
            assertEquals(50, results.ElementAt(1).Value);
        }

        [Test]
        public void TestQueueExhaustion()
        {
            Analyzer a = new MockAnalyzer(Random);
            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, SuggesterOptions.EXACT_FIRST, 256, -1, true);

            suggester.Build(new InputArrayIterator(new Input[] {
                new Input("a", 2),
                new Input("a b c", 3),
                new Input("a c a", 1),
                new Input("a c b", 1),
            }));

            suggester.DoLookup("a", false, 4);
        }

        [Test]
        public void TestExactFirstMissingResult()
        {

            Analyzer a = new MockAnalyzer(Random);

            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, SuggesterOptions.EXACT_FIRST, 256, -1, true);

            suggester.Build(new InputArrayIterator(new Input[] {
                new Input("a", 5),
                new Input("a b", 3),
                new Input("a c", 4),
            }));

            assertEquals(3, suggester.Count);
            IList<Lookup.LookupResult> results = suggester.DoLookup("a", false, 3);
            assertEquals(3, results.size());
            assertEquals("a", results.ElementAt(0).Key);
            assertEquals(5, results.ElementAt(0).Value);
            assertEquals("a c", results.ElementAt(1).Key);
            assertEquals(4, results.ElementAt(1).Value);
            assertEquals("a b", results.ElementAt(2).Key);
            assertEquals(3, results.ElementAt(2).Value);

            // Try again after save/load:
            DirectoryInfo tmpDir = CreateTempDir("AnalyzingSuggesterTest");
            tmpDir.Create();

            FileInfo path = new FileInfo(Path.Combine(tmpDir.FullName, "suggester"));

            Stream os = new FileStream(path.FullName, FileMode.OpenOrCreate);
            suggester.Store(os);
            os.Dispose();

            Stream @is = new FileStream(path.FullName, FileMode.Open);
            suggester.Load(@is);
            @is.Dispose();

            assertEquals(3, suggester.Count);
            results = suggester.DoLookup("a", false, 3);
            assertEquals(3, results.size());
            assertEquals("a", results.ElementAt(0).Key);
            assertEquals(5, results.ElementAt(0).Value);
            assertEquals("a c", results.ElementAt(1).Key);
            assertEquals(4, results.ElementAt(1).Value);
            assertEquals("a b", results.ElementAt(2).Key);
            assertEquals(3, results.ElementAt(2).Value);
        }

        internal class TestDupSurfaceFormsMissingResultsTokenStreamComponents : TokenStreamComponents
        {
            private readonly AnalyzingSuggesterTest outerInstance;
            public TestDupSurfaceFormsMissingResultsTokenStreamComponents(AnalyzingSuggesterTest outerInstance, Tokenizer tokenizer)
                : base(tokenizer)
            {
                this.outerInstance = outerInstance;
            }

            public override TokenStream TokenStream
            {
                get
                {
                    return new CannedTokenStream(new Token[] {
                        NewToken("hairy", 1, 1),
                        NewToken("smelly", 0, 1),
                        NewToken("dog", 1, 1),
                    });
                }
            }

            protected override void SetReader(TextReader reader)
            {
            }
        }

        internal class TestDupSurfaceFormsMissingResultsAnalyzer : Analyzer
        {
            private readonly AnalyzingSuggesterTest outerInstance;
            public TestDupSurfaceFormsMissingResultsAnalyzer(AnalyzingSuggesterTest outerInstance)
            {
                this.outerInstance = outerInstance;
            }
            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
                return new TestDupSurfaceFormsMissingResultsTokenStreamComponents(outerInstance, tokenizer);
            }
        }

        [Test]
        public void TestDupSurfaceFormsMissingResults()
        {
            Analyzer a = new TestDupSurfaceFormsMissingResultsAnalyzer(this);

            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1, true);

            suggester.Build(new InputArrayIterator(Shuffle(
                      new Input("hambone", 6),
                      new Input("nellie", 5))));

            IList<Lookup.LookupResult> results = suggester.DoLookup("nellie", false, 2);
            assertEquals(2, results.size());
            assertEquals("hambone", results.ElementAt(0).Key);
            assertEquals(6, results.ElementAt(0).Value);
            assertEquals("nellie", results.ElementAt(1).Key);
            assertEquals(5, results.ElementAt(1).Value);

            // Try again after save/load:
            DirectoryInfo tmpDir = CreateTempDir("AnalyzingSuggesterTest");
            tmpDir.Create();

            FileInfo path = new FileInfo(Path.Combine(tmpDir.FullName, "suggester"));

            Stream os = new FileStream(path.FullName, FileMode.OpenOrCreate);
            suggester.Store(os);
            os.Dispose();

            Stream @is = new FileStream(path.FullName, FileMode.Open);
            suggester.Load(@is);
            @is.Dispose();

            results = suggester.DoLookup("nellie", false, 2);
            assertEquals(2, results.size());
            assertEquals("hambone", results.ElementAt(0).Key);
            assertEquals(6, results.ElementAt(0).Value);
            assertEquals("nellie", results.ElementAt(1).Key);
            assertEquals(5, results.ElementAt(1).Value);
        }
        internal class TestDupSurfaceFormsMissingResults2TokenStreamComponents : TokenStreamComponents
        {
            internal int count;
            public TestDupSurfaceFormsMissingResults2TokenStreamComponents(Tokenizer tokenizer)
                : base(tokenizer)
            { }

            public override TokenStream TokenStream
            {
                get
                {
                    if (count == 0)
                    {
                        count++;
                        return new CannedTokenStream(new Token[] {
                            NewToken("p", 1, 1),
                            NewToken("q", 1, 1),
                            NewToken("r", 0, 1),
                            NewToken("s", 0, 1),
                        });
                    }
                    else
                    {
                        return new CannedTokenStream(new Token[] {
                            NewToken("p", 1, 1),
                        });
                    }
                }
            }

            protected override void SetReader(TextReader reader)
            {
            }
        }

        internal class TestDupSurfaceFormsMissingResults2Analyzer : Analyzer
        {
            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
                return new TestDupSurfaceFormsMissingResults2TokenStreamComponents(tokenizer);
            }
        }

        [Test]
        public void TestDupSurfaceFormsMissingResults2()
        {
            Analyzer a = new TestDupSurfaceFormsMissingResults2Analyzer();

            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1, true);

            suggester.Build(new InputArrayIterator(new Input[] {
                new Input("a", 6),
                new Input("b", 5),
            }));

            IList<Lookup.LookupResult> results = suggester.DoLookup("a", false, 2);
            assertEquals(2, results.size());
            assertEquals("a", results.ElementAt(0).Key);
            assertEquals(6, results.ElementAt(0).Value);
            assertEquals("b", results.ElementAt(1).Key);
            assertEquals(5, results.ElementAt(1).Value);

            // Try again after save/load:
            DirectoryInfo tmpDir = CreateTempDir("AnalyzingSuggesterTest");
            tmpDir.Create();

            FileInfo path = new FileInfo(Path.Combine(tmpDir.FullName, "suggester"));

            Stream os = new FileStream(path.FullName, FileMode.OpenOrCreate);
            suggester.Store(os);
            os.Dispose();

            Stream @is = new FileStream(path.FullName, FileMode.Open);
            suggester.Load(@is);
            @is.Dispose();

            results = suggester.DoLookup("a", false, 2);
            assertEquals(2, results.size());
            assertEquals("a", results.ElementAt(0).Key);
            assertEquals(6, results.ElementAt(0).Value);
            assertEquals("b", results.ElementAt(1).Key);
            assertEquals(5, results.ElementAt(1).Value);
        }

        internal class Test0ByteKeysTokenStreamComponents : TokenStreamComponents
        {
            internal int tokenStreamCounter = 0;
            internal TokenStream[] tokenStreams = new TokenStream[] {
              new CannedBinaryTokenStream(new BinaryToken[] {
                  NewToken(new BytesRef(new byte[] {0x0, 0x0, 0x0})),
                }),
              new CannedBinaryTokenStream(new BinaryToken[] {
                  NewToken(new BytesRef(new byte[] {0x0, 0x0})),
                }),
              new CannedBinaryTokenStream(new BinaryToken[] {
                  NewToken(new BytesRef(new byte[] {0x0, 0x0, 0x0})),
                }),
              new CannedBinaryTokenStream(new BinaryToken[] {
                  NewToken(new BytesRef(new byte[] {0x0, 0x0})),
                }),
            };

            public Test0ByteKeysTokenStreamComponents(Tokenizer tokenizer)
                : base(tokenizer)
            { }

            public override TokenStream TokenStream
            {
                get
                {
                    TokenStream result = tokenStreams[tokenStreamCounter];
                    tokenStreamCounter++;
                    return result;
                }
            }

            protected override void SetReader(TextReader reader)
            {
            }
        }

        internal class Test0ByteKeysAnalyzer : Analyzer
        {
            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
                return new Test0ByteKeysTokenStreamComponents(tokenizer);
            }
        }

        [Test]
        public void Test0ByteKeys()
        {
            Analyzer a = new Test0ByteKeysAnalyzer();

            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1, true);

            suggester.Build(new InputArrayIterator(new Input[] {
                new Input("a a", 50),
                new Input("a b", 50),
            }));
        }

        [Test]
        public void TestDupSurfaceFormsMissingResults3()
        {
            Analyzer a = new MockAnalyzer(Random);
            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, SuggesterOptions.PRESERVE_SEP, 256, -1, true);
            suggester.Build(new InputArrayIterator(new Input[] {
                new Input("a a", 7),
                new Input("a a", 7),
                new Input("a c", 6),
                new Input("a c", 3),
                new Input("a b", 5),
            }));
            assertEquals("[a a/7, a c/6, a b/5]", suggester.DoLookup("a", false, 3).toString());
        }

        [Test]
        public void TestEndingSpace()
        {
            Analyzer a = new MockAnalyzer(Random);
            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, SuggesterOptions.PRESERVE_SEP, 256, -1, true);
            suggester.Build(new InputArrayIterator(new Input[] {
                new Input("i love lucy", 7),
                new Input("isla de muerta", 8),
            }));
            assertEquals("[isla de muerta/8, i love lucy/7]", suggester.DoLookup("i", false, 3).toString());
            assertEquals("[i love lucy/7]", suggester.DoLookup("i ", false, 3).toString());
        }

        internal class TestTooManyExpressionsTokenStreamComponents : TokenStreamComponents
        {
            public TestTooManyExpressionsTokenStreamComponents(Tokenizer tokenizer)
                : base(tokenizer)
            { }

            public override TokenStream TokenStream
            {
                get
                {
                    Token a = new Token("a", 0, 1);
                    a.PositionIncrement = (1);
                    Token b = new Token("b", 0, 1);
                    b.PositionIncrement = (0);
                    return new CannedTokenStream(new Token[] { a, b });
                }
            }

            protected override void SetReader(TextReader reader)
            {
            }
        }
        internal class TestTooManyExpressionsAnalyzer : Analyzer
        {
            protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
            {
                Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
                return new TestTooManyExpressionsTokenStreamComponents(tokenizer);
            }
        }

        [Test]
        public void TestTooManyExpansions()
        {
            Analyzer a = new TestTooManyExpressionsAnalyzer();

            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, 1, true);
            suggester.Build(new InputArrayIterator(new Input[] { new Input("a", 1) }));

            assertEquals("[a/1]", suggester.DoLookup("a", false, 1).toString());
        }

        [Test]
        public void TestIllegalLookupArgument()
        {
            Analyzer a = new MockAnalyzer(Random);
            AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1, true);
            suggester.Build(new InputArrayIterator(new Input[] {
                new Input("а где Люси?", 7),
            }));
            try
            {
                suggester.DoLookup("а\u001E", false, 3);
                fail("should throw IllegalArgumentException");
            }
            catch (ArgumentException /*e*/)
            {
                // expected
            }
            try
            {
                suggester.DoLookup("а\u001F", false, 3);
                fail("should throw IllegalArgumentException");
            }
            catch (ArgumentException /*e*/)
            {
                // expected
            }
        }

        internal static IEnumerable<Input> Shuffle(params Input[] values)
        {
            IList<Input> asList = new List<Input>(values.Length);
            foreach (Input value in values)
            {
                asList.Add(value);
            }
            Collections.Shuffle(asList);
            return asList;
        }
    }
}
