Fixes #851: Resolve encoding issues during import/export with Python 3

This commit is contained in:
Jeremy Stretch 2017-02-01 11:49:54 -05:00
parent 5eb3c1a67b
commit 0eba5a0de3
2 changed files with 25 additions and 13 deletions

View File

@ -236,14 +236,15 @@ class CSVDataField(forms.CharField):
if not self.help_text: if not self.help_text:
self.help_text = 'Enter one line per record in CSV format.' self.help_text = 'Enter one line per record in CSV format.'
def utf_8_encoder(self, unicode_csv_data):
for line in unicode_csv_data:
yield line.encode('utf-8')
def to_python(self, value): def to_python(self, value):
# Return a list of dictionaries, each representing an individual record """
Return a list of dictionaries, each representing an individual record
"""
# Python 2's csv module has problems with Unicode
if not isinstance(value, str):
value = value.encode('utf-8')
records = [] records = []
reader = csv.reader(self.utf_8_encoder(value.splitlines())) reader = csv.reader(value.splitlines())
for i, row in enumerate(reader, start=1): for i, row in enumerate(reader, start=1):
if row: if row:
if len(row) < len(self.columns): if len(row) < len(self.columns):

View File

@ -1,15 +1,26 @@
import six
def csv_format(data): def csv_format(data):
""" """
Encapsulate any data which contains a comma within double quotes. Encapsulate any data which contains a comma within double quotes.
""" """
csv = [] csv = []
for d in data: for value in data:
if d in [None, False]:
# Represent None or False with empty string
if value in [None, False]:
csv.append(u'') csv.append(u'')
elif type(d) not in (str, unicode): continue
csv.append(u'{}'.format(d))
elif u',' in d: # Force conversion to string first so we can check for any commas
csv.append(u'"{}"'.format(d)) if not isinstance(value, six.string_types):
value = u'{}'.format(value)
# Double-quote the value if it contains a comma
if u',' in value:
csv.append(u'"{}"'.format(value))
else: else:
csv.append(d) csv.append(u'{}'.format(value))
return u','.join(csv) return u','.join(csv)