#!/usr/bin/python

import sys
import iopro
import unittest
from generate import generate_dataset, IntIter, MissingValuesIter, FixedWidthIter
import StringIO
import numpy
import gzip
import os


class TestTextAdapter(unittest.TestCase):

    num_records = 100000

    def assert_equality(self, left, right):
        try:
            self.assertTrue(left == right)
        except AssertionError:
            raise AssertionError('FAIL: {0} != {1}'.format(left, right))

    # Basic parsing tests
    def test_string_parsing(self):
        data = StringIO.StringIO('1,2,3\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter=',')
        adapter.set_field_types({0:'S5', 1:'S5', 2:'S5'})
        data = StringIO.StringIO('1,2,3\n')

    def test_no_whitespace_stripping(self):
        data = StringIO.StringIO('1  ,2  ,3  \n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter=',')
        adapter.set_field_types({0:'S3', 1:'S3', 2:'S3'})
        self.assert_equality(adapter[0].item(), ("1  ", "2  ", "3  "))

        data = StringIO.StringIO('  1,  2,  3\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter=',')
        adapter.set_field_types({0:'S3', 1:'S3', 2:'S3'})
        self.assert_equality(adapter[0].item(), ("  1", "  2", "  3"))

        data = StringIO.StringIO('  1  ,  2  ,  3  \n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter=',')
        adapter.set_field_types({0:'S5', 1:'S5', 2:'S5'})
        self.assert_equality(adapter[0].item(), ("  1  ", "  2  ", "  3  "))

        data = StringIO.StringIO('\t1\t,\t2\t,\t3\t\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter=',')
        adapter.set_field_types({0:'S3', 1:'S3', 2:'S3'})
        self.assert_equality(adapter[0].item(), ("\t1\t", "\t2\t", "\t3\t"))

    def test_quoted_whitespace(self):
        data = StringIO.StringIO('"1  ","2  ","3  "\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter=',')
        adapter.set_field_types({0:'S3', 1:'S3', 2:'S3'})
        self.assert_equality(adapter[0].item(), ("1  ", "2  ", "3  "))

        data = StringIO.StringIO('"\t1\t"\t"\t2\t"\t"\t3\t"\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter='\t')
        adapter.set_field_types({0:'S3', 1:'S3', 2:'S3'})
        self.assert_equality(adapter[0].item(), ("\t1\t", "\t2\t", "\t3\t"))

    def test_fixed_simple(self):
        # TODO: fix this test on 32-bit and on Windows
        if tuple.__itemsize__ == 4:
            # This test does not work on 32-bit, so we skip it
            return
        if sys.platform == 'win32':
            # This test does not work on Windows
            return
        data = StringIO.StringIO("  1  2  3\n  4  5 67\n890123  4")
        adapter = iopro.FixedWidthTextAdapter(data, 3, infer_types=False, field_names=False)
        adapter.set_field_types({0:'i', 1:'i', 2:'i'})

        control = [(1, 2, 3), (4, 5, 67), (890, 123, 4)]
        self.assert_(control == adapter[:].tolist())

    def test_spaces_around_numeric_values(self):
        data = StringIO.StringIO(' 1 , 2.0 , 3.3 \n  4  ,  5.0  ,  6.6  ')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter=',')
        adapter.set_field_types({0:'u4', 1:'i4', 2:'f8'})

        self.assert_equality(adapter[0].item(), (1, 2, 3.3))
        self.assert_equality(adapter[1].item(), (4, 5, 6.6))


    def test_slicing(self):

        data = StringIO.StringIO()
        generate_dataset(data, IntIter(), ',', self.num_records)
        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=False)
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})

        self.assert_equality(adapter[0].item(), (0, 1, 2, 3, 4))
        expected_values = [((self.num_records-1)*5)+x for x in range(5)]
        self.assert_equality(adapter[self.num_records-1].item(), tuple(expected_values))

        #adapter.create_index()
        #self.assert_equality(adapter[-1].item(), tuple(expected_values))

        self.assert_equality(adapter['f0'][0].item(), (0,))
        self.assert_equality(adapter['f4'][1].item(), (9,))
        #self.assert_equality(adapter[self.num_records-1]['f4'], (self.num_records*5)-1)
 
        array = adapter[:]
        record = [x for x in range(0, 5)]
        self.assert_equality(array.size, self.num_records)
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        array = adapter[:-1]
        record = [x for x in range(0, 5)]
        self.assert_equality(array.size, self.num_records-1)
        for i in range(0, self.num_records-1):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        array = adapter[0:10]
        self.assert_equality(array.size, 10)
        record = [x for x in range(0, 5)]
        for i in range(0, 10):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        array = adapter[1:]
        self.assert_equality(array.size, self.num_records-1)
        record = [x for x in range(5, 10)]
        for i in range(0, self.num_records-1):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        array = adapter[0:10:2]
        self.assert_equality(array.size, 5)
        record = [x for x in range(0, 5)]
        for i in range(0, 5):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+10 for x in record]

        array = adapter[['f0', 'f4']][:]
        record = [0, 4]
        self.assert_equality(array.size, self.num_records)
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        adapter.field_filter = [0, 'f4']
        array = adapter[:]
        record = [0, 4]
        self.assert_equality(array.size, self.num_records)
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        adapter.field_filter = None
        array = adapter[:]
        record = [0, 1, 2, 3, 4]
        self.assert_equality(array.size, self.num_records)
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        try:
            adapter[self.num_records]
        except iopro.AdapterIndexError:
            pass
        else:
            self.fail('AdaperIndexError not thrown')

        try:
            adapter[0:self.num_records+1]
        except iopro.AdapterIndexError:
            pass
        else:
            self.fail('AdaperIndexError not thrown')


    def test_converters(self):

        data = StringIO.StringIO()
        generate_dataset(data, IntIter(), ',', self.num_records)
        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False)
        #adapter.set_field_types({0:'u4', 1:'u4', 2:'u4', 3:'u4', 4:'u4'})

        def increment(input_str):
            return int(input_str) + 1

        def double(input_str):
            return int(input_str) + int(input_str)

        if sys.platform == 'win32' and tuple.__itemsize__ == 8:
            # TODO: there problems below here 64-bit Windows, I get
            # OverflowError: can't convert negative value to unigned PY_LONG_LONG
            return

        adapter.set_converter(0, increment)
        adapter.set_converter('f1', double)

        array = adapter[:]

        self.assert_equality(array.size, self.num_records)
        
        record = [1, 2, 2, 3, 4]
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record[0] += 5
            record[1] = (10 * (i+1)) + 2
            record[2] += 5
            record[3] += 5
            record[4] += 5


    def test_missing_fill_values(self):

        data = StringIO.StringIO()
        generate_dataset(data, MissingValuesIter(), ',', self.num_records)

        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=False)
        adapter.set_field_types({'f0':'u4', 1:'u4', 2:'u4', 3:'u4', 'f4':'u4'})
        adapter.set_missing_values({0:['NA', 'NaN'], 'f4':['xx','inf']})
        adapter.set_fill_values({0:99, 4:999})

        array = adapter[:]

        self.assert_equality(array.size, self.num_records)
        
        record = [x for x in range(0, 5)]
        for i in range(0, self.num_records):
            if i % 4 == 0 or i % 4 == 1:
                record[0] = 99
                record[4] = 999
            else:
                record[0] = record[1] - 1
                record[4] = record[3] + 1
            self.assert_equality(array[i].item(), tuple(record))
            record[1] += 5
            record[2] += 5
            record[3] += 5

        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=True)
        adapter.set_missing_values({0:['NA', 'NaN'], 4:['xx','inf']})

        array = adapter[:]

        self.assert_equality(array.size, self.num_records)
        
        record = [x for x in range(0, 5)]
        for i in range(0, self.num_records):
            if i % 4 == 0 or i % 4 == 1:
                record[0] = 0 
                record[4] = 0
            else:
                record[0] = record[1] - 1
                record[4] = record[3] + 1
            self.assert_equality(array[i].item(), tuple(record))
            record[1] += 5
            record[2] += 5
            record[3] += 5


    def test_fixed_width(self):

        data = StringIO.StringIO()
        generate_dataset(data, FixedWidthIter(), '', self.num_records)
        adapter = iopro.FixedWidthTextAdapter(data, [2,3,4,5,6], field_names=False, infer_types=False)
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})

        array = adapter[:]

        self.assert_equality(array.size, self.num_records)
        
        record = [0, 0, 0, 0, 0]
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+1 for x in record]
            if record[0] == 100:
                record[0] = 0
            if record[1] == 1000:
                record[1] = 0
            if record[2] == 10000:
                record[2] = 0
            if record[3] == 100000:
                record[3] = 0
            if record[4] == 1000000:
                record[4] = 0


    def test_regex(self):

        data = StringIO.StringIO()
        generate_dataset(data, IntIter(), ',', self.num_records)
        adapter = iopro.RegexTextAdapter(data, '([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*)\n', field_names=False, infer_types=False)
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})        

        array = adapter[:]

        self.assert_equality(array.size, self.num_records)
        
        record = [x for x in range(0, 5)]
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]


    def test_index(self):
        if sys.platform == 'win32':
            # TODO: this test fails on Windows because of file lock problems
            return

        num_records = 100000
        expected_values = [((num_records-1)*5) + x for x in range(5)]

        data = StringIO.StringIO()
        generate_dataset(data, IntIter(), ',', num_records)
        
        # test explicit index building
        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=False)
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})
        adapter.create_index()

        self.assert_equality(adapter[0].item(), tuple([(0*5) + x for x in range(5)]))
        self.assert_equality(adapter[10].item(), tuple([(10*5) + x for x in range(5)]))
        self.assert_equality(adapter[100].item(), tuple([(100*5) + x for x in range(5)]))
        self.assert_equality(adapter[1000].item(), tuple([(1000*5) + x for x in range(5)]))
        self.assert_equality(adapter[10000].item(), tuple([(10000*5) + x for x in range(5)]))
        self.assert_equality(adapter[num_records - 1].item(), tuple([((num_records - 1)*5) + x for x in range(5)]))
        #self.assert_equality(adapter[-1].item(), tuple(expected_values))

        # test implicitly creating disk index on the fly
        if os.path.exists('test.idx'):
            os.remove('test.idx')
        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=False, index_name='test.idx')
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})
        adapter.to_array()

        self.assert_equality(adapter[0].item(), tuple([(0*5) + x for x in range(5)]))
        self.assert_equality(adapter[10].item(), tuple([(10*5) + x for x in range(5)]))
        self.assert_equality(adapter[100].item(), tuple([(100*5) + x for x in range(5)]))
        self.assert_equality(adapter[1000].item(), tuple([(1000*5) + x for x in range(5)]))
        self.assert_equality(adapter[10000].item(), tuple([(10000*5) + x for x in range(5)]))
        self.assert_equality(adapter[num_records - 1].item(), tuple([((num_records - 1)*5) + x for x in range(5)]))
        #self.assert_equality(adapter[-1].item(), tuple(expected_values))

        adapter.close()

        # test loading disk index
        adapter2 = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=False, index_name='test.idx')
        adapter2.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})

        self.assert_equality(adapter2[0].item(), tuple([(0*5) + x for x in range(5)]))
        self.assert_equality(adapter2[10].item(), tuple([(10*5) + x for x in range(5)]))
        self.assert_equality(adapter2[100].item(), tuple([(100*5) + x for x in range(5)]))
        self.assert_equality(adapter2[1000].item(), tuple([(1000*5) + x for x in range(5)]))
        self.assert_equality(adapter2[10000].item(), tuple([(10000*5) + x for x in range(5)]))
        self.assert_equality(adapter2[num_records - 1].item(), tuple([((num_records - 1)*5) + x for x in range(5)]))
        #self.assert_equality(adapter2[-1].item(), tuple(expected_values))

        adapter.close()

        os.remove('test.idx')


    def test_gzip_index(self):

        num_records = 1000000

        data = StringIO.StringIO()
        generate_dataset(data, IntIter(), ',', num_records)

        dataz = StringIO.StringIO()
        gzip_output = gzip.GzipFile(fileobj=dataz, mode='wb')
        gzip_output.write(data.getvalue())
        gzip_output.close()

        # test explicit index building
        adapter = iopro.CSVTextAdapter(dataz, compression='gzip', delimiter=',', field_names=False, infer_types=False)
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})
        adapter.create_index()
    
        self.assert_equality(adapter[0].item(), tuple([(0*5) + x for x in range(5)]))
        self.assert_equality(adapter[10].item(), tuple([(10*5) + x for x in range(5)]))
        self.assert_equality(adapter[100].item(), tuple([(100*5) + x for x in range(5)]))
        self.assert_equality(adapter[1000].item(), tuple([(1000*5) + x for x in range(5)]))
        self.assert_equality(adapter[10000].item(), tuple([(10000*5) + x for x in range(5)]))
        self.assert_equality(adapter[100000].item(), tuple([(100000*5) + x for x in range(5)]))
        self.assert_equality(adapter[num_records - 1].item(), tuple([((num_records - 1)*5) + x for x in range(5)]))
        #self.assert_equality(adapter[-1].item(), tuple(expected_values))

        # test 'trouble' records that have caused crashes in the past
        self.assert_equality(adapter[290000].item(), tuple([(290000*5) + x for x in range(5)]))
        self.assert_equality(adapter[818000].item(), tuple([(818000*5) + x for x in range(5)]))

        # test implicitly creating disk index on the fly
        # JNB: not implemented yet
        '''adapter = iopro.CSVTextAdapter(dataz, compression='gzip', delimiter=',', field_names=False, infer_types=False, indexing=True, index_filename='test.idx')
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})
        adapter.to_array()

        self.assert_equality(adapter[0].item(), tuple([(0*5) + x for x in range(5)]))
        self.assert_equality(adapter[10].item(), tuple([(10*5) + x for x in range(5)]))
        self.assert_equality(adapter[100].item(), tuple([(100*5) + x for x in range(5)]))
        self.assert_equality(adapter[1000].item(), tuple([(1000*5) + x for x in range(5)]))
        self.assert_equality(adapter[10000].item(), tuple([(10000*5) + x for x in range(5)]))
        self.assert_equality(adapter[100000].item(), tuple([(100000*5) + x for x in range(5)]))
        self.assert_equality(adapter[num_records - 1].item(), tuple([((num_records - 1)*5) + x for x in range(5)]))
        #self.assert_equality(adapter[-1].item(), tuple(expected_values))

        # test 'trouble' records that have caused crashes in the past
        self.assert_equality(adapter[290000].item(), tuple([(290000*5) + x for x in range(5)]))
        self.assert_equality(adapter[818000].item(), tuple([(818000*5) + x for x in range(5)]))

        # test loading disk index
        adapter2 = iopro.CSVTextAdapter(dataz, compression='gzip', delimiter=',', field_names=False, infer_types=False, indexing=True, index_filename='test.idx')
        adapter2.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})

        self.assert_equality(adapter2[0].item(), tuple([(0*5) + x for x in range(5)]))
        self.assert_equality(adapter2[10].item(), tuple([(10*5) + x for x in range(5)]))
        self.assert_equality(adapter2[100].item(), tuple([(100*5) + x for x in range(5)]))
        self.assert_equality(adapter2[1000].item(), tuple([(1000*5) + x for x in range(5)]))
        self.assert_equality(adapter2[10000].item(), tuple([(10000*5) + x for x in range(5)]))
        self.assert_equality(adapter2[100000].item(), tuple([(100000*5) + x for x in range(5)]))
        self.assert_equality(adapter2[num_records - 1].item(), tuple([((num_records - 1)*5) + x for x in range(5)]))
        #self.assert_equality(adapter[-1].item(), tuple(expected_values))

        # test 'trouble' records that have caused crashes in the past
        self.assert_equality(adapter2[290000].item(), tuple([(290000*5) + x for x in range(5)]))
        self.assert_equality(adapter2[818000].item(), tuple([(818000*5) + x for x in range(5)]))

        os.remove('test.idx')'''


    # JNB: fix this unit test
    def xtest_header_footer(self):

        data = StringIO.StringIO()
        generate_dataset(data, IntIter(), ',', self.num_records)
        adapter = iopro.TextAdapter(data, header=100, footer=100, field_names=False, infer_types=False)
        adapter.setDelimiter(',')
        adapter.set_field_types(dict(zip(range(5), ['u4']*5)))
        adapter.create_index()

        self.assert_equality(adapter[0].item(), (500, 501, 502, 503, 504))
        self.assert_equality(adapter[-1].item(), (12499495, 12499496, 12499497, 12499498, 12499498))


    def test_delimiter_sniffing(self):

        data = StringIO.StringIO('1,2,3\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False)
        self.assert_equality(adapter[0].item(), (1,2,3))

        data = StringIO.StringIO('1 2 3\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False)
        self.assert_equality(adapter[0].item(), (1,2,3))

        data = StringIO.StringIO('1\t2\t3\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False)
        self.assert_equality(adapter[0].item(), (1,2,3))

        data = StringIO.StringIO('1x2x3\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False)
        self.assert_equality(adapter[0].item(), (1,2,3))


    def test_auto_type_inference(self):

        data = StringIO.StringIO('0,1,2,3,4\n5.5,6,7,8,9\n10,11,12,13,14a\n15,16,xxx,18,19')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=True)
        array = adapter.to_array()
        self.assert_equality(array.dtype.fields['f0'][0], numpy.dtype('float64'))
        self.assert_equality(array.dtype.fields['f1'][0], numpy.dtype('uint64'))
        self.assert_equality(array.dtype.fields['f2'][0], numpy.dtype('O'))
        self.assert_equality(array.dtype.fields['f3'][0], numpy.dtype('uint64'))
        self.assert_equality(array.dtype.fields['f4'][0], numpy.dtype('O'))

        data = StringIO.StringIO('0,1,2,3,4\n5.5,6,7,8,9\n10,11,12,13,14a\n15,16,xxx,18,19')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=True)
        self.assert_equality(adapter[0].dtype.fields['f0'][0], numpy.dtype('uint64'))
        self.assert_equality(adapter[1:3].dtype.fields['f0'][0], numpy.dtype('float64'))
        self.assert_equality(adapter[3].dtype.fields['f4'][0], numpy.dtype('uint64'))
        self.assert_equality(adapter[:].dtype.fields['f3'][0], numpy.dtype('uint64'))
        self.assert_equality(adapter[-1].dtype.fields['f2'][0], numpy.dtype('O'))
        self.assert_equality(adapter[2].dtype.fields['f4'][0], numpy.dtype('O'))

    def test_64bit_ints(self):
        
        data = StringIO.StringIO(str((2**63)-1) + ',' + str(((2**63)-1)*-1) + ',' + str((2**64)-1))
        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=False)
        adapter.set_field_types({0:'i8', 1:'i8', 2:'u8'})
        array = adapter.to_array()
        self.assert_equality(array[0].item(), ((2**63)-1, ((2**63)-1)*-1, (2**64)-1))
 
    def test_adapter_factory(self):
        data = StringIO.StringIO("1,2,3")
        adapter = iopro.text_adapter(data, "csv", delimiter=',', field_names=False, infer_types=False)
        self.assert_(isinstance(adapter, iopro.CSVTextAdapter))

        self.assertRaises(iopro.AdapterException, iopro.text_adapter, data, "foobar")

    def test_field_names(self):

        # Test for ignoring of extra fields
        data = StringIO.StringIO('f0,f1\n0,1,2\n3,4,5')
        adapter = iopro.text_adapter(data, 'csv', delimiter=',', field_names=True)
        array = adapter.to_array()
        self.assert_equality(array.dtype.names, ('f0', 'f1'))
        self.assert_equality(array[0].item(), (0,1))
        self.assert_equality(array[1].item(), (3,4))

        # Test for duplicate field names
        data = StringIO.StringIO('f0,field,field\n0,1,2\n,4,5')
        adapter = iopro.text_adapter(data, 'csv', delimiter=',', field_names=True, infer_types=False)
        adapter.set_field_types({0:'u4', 1:'u4', 2:'u4'})
        array = adapter.to_array()
        self.assert_equality(array.dtype.names, ('f0', 'field', 'field1'))

    def test_float_conversion(self):
        data = StringIO.StringIO('10,1.333,-1.23,10.0E+2,999.9e-2')
        adapter = iopro.text_adapter(data, field_names=False, infer_types=False)
        adapter.set_field_types(dict(zip(range(5), ['f8']*5)))
        array = adapter[0]
        #self.assert_equality(array[0].item(), (10.0,1.333,-1.23,1000.0,9.999))
        self.assertAlmostEqual(array[0][0], 10.0)
        self.assertAlmostEqual(array[0][1], 1.333)
        self.assertAlmostEqual(array[0][2], -1.23)
        self.assertAlmostEqual(array[0][3], 1000.0)
        self.assertAlmostEqual(array[0][4], 9.999)

    def test_generators(self):

        def int_generator(num_recs):
            for i in range(num_recs):
                yield ','.join([str(i*5), str(i*5+1), str(i*5+2), str(i*5+3), str(i*5+4)])

        adapter = iopro.text_adapter(int_generator(self.num_records), field_names=False)
        array = adapter[:]

        self.assert_equality(array.size, self.num_records)
        
        record = [x for x in range(0, 5)]
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record[0] += 5
            record[1] += 5
            record[2] += 5
            record[3] += 5
            record[4] += 5

    def test_comments(self):

        data = StringIO.StringIO('1,2,3\n#4,5,6')
        adapter = iopro.text_adapter(data, field_names=False)
        array = adapter[:]
        self.assert_equality(array.size, 1)
        self.assert_equality(array[0].item(), (1,2,3))

        data = StringIO.StringIO('1,2,3\n#4,5,6')
        adapter = iopro.text_adapter(data, field_names=False, comment=None)
        array = adapter[:]
        self.assert_equality(array.size, 2)
        self.assert_equality(array[0].item(), ('1',2,3))
        self.assert_equality(array[1].item(), ('#4',5,6))


def run(verbosity=1, num_records=100000):
    if num_records < 10:
        raise ValueError('number of records for generated datasets must be at least 10')
    TestTextAdapter.num_records = num_records
    suite = unittest.TestLoader().loadTestsFromTestCase(TestTextAdapter)
    return unittest.TextTestRunner(verbosity=verbosity).run(suite)


if __name__ == '__main__':
    run()
