diff --git a/changes/77.changes b/changes/77.changes new file mode 100644 index 0000000..17cd7b7 --- /dev/null +++ b/changes/77.changes @@ -0,0 +1,2 @@ +Refactor handling on empty values for multivalued fields. + diff --git a/search/management/commands/import_data_csv.py b/search/management/commands/import_data_csv.py index d13a984..5db431c 100644 --- a/search/management/commands/import_data_csv.py +++ b/search/management/commands/import_data_csv.py @@ -69,22 +69,41 @@ def set_empty_fields(self, solr_record: dict): def set_empty_field(self, solr_record: dict, sf: Field): # look for empty fields, empty multi-value fields, and empty multi-value codes and set them - if (sf.field_id not in solr_record and sf != 'default_fmt') or (solr_record[sf.field_id] == '') or \ - (isinstance(solr_record[sf.field_id], list) and len(solr_record[sf.field_id]) < 1) or \ - (isinstance(solr_record[sf.field_id], list) and solr_record[sf.field_id][0] == ''): - if sf.default_export_value: - default_fmt = sf.default_export_value.split('|') - if default_fmt[0] in ['str', 'date']: - solr_record[sf.field_id] = str(default_fmt[1]) - elif default_fmt[0] == 'int': - solr_record[sf.field_id] = int(default_fmt[1]) - elif default_fmt[0] == 'float': - solr_record[sf.field_id] = float(default_fmt[1]) - else: - solr_record[sf.field_id] = '' - if sf.solr_field_is_coded: - solr_record[f"{sf.field_id}_en"] = "-" - solr_record[f"{sf.field_id}_fr"] = "-" + val = solr_record.get(sf.field_id) + is_empty = ( + (sf.field_id not in solr_record and sf.field_id != 'default_fmt') or + val is None or val == "" or + (isinstance(val, list) and (len(val) == 0 or all(str(i).strip() == '' for i in val))) + ) + + if is_empty: + fallback_value = '' + + if sf.default_export_value and '|' in sf.default_export_value: + try: + data_type, default_value = sf.default_export_value.split('|', 1) + + if data_type in ['str', 'date']: + fallback_value = default_value + elif data_type == 'int': + fallback_value = int(default_value) + elif data_type == 'float': + fallback_value = float(default_value) + except (ValueError, TypeError): + fallback_value = '' + + # check for multivalued field + is_multivalued = getattr(sf, 'solr_field_multivalued', False) + final_value = [fallback_value] if is_multivalued else fallback_value + + solr_record[sf.field_id] = final_value + + # check for localized fields + is_bilingual = getattr(sf, 'solr_field_lang', '') == 'bi' + + if sf.solr_field_is_coded or is_bilingual: + solr_record[f"{sf.field_id}_en"] = final_value + solr_record[f"{sf.field_id}_fr"] = final_value def handle(self, *args, **options): @@ -383,7 +402,7 @@ def handle(self, *args, **options): solr_record[csv_field + '_en'] = "-" solr_record[csv_field + '_fr'] = "-" - # Ensure all empty CSV fields are set to appropriate or default values + # Ensure all empty CSV fields are set to appropriate or default values solr_record = self.set_empty_fields(solr_record) diff --git a/search/tasks.py b/search/tasks.py index 5189027..bb427cf 100644 --- a/search/tasks.py +++ b/search/tasks.py @@ -21,10 +21,14 @@ def cache_search_results_file(cached_filename: str, sr: SolrResponse, rows=0, fi with open(f"{cached_filename}.tmp", 'w', newline='', encoding='utf8') as csv_file: cache_writer = csv.writer(csv_file, dialect='excel', quoting=csv.QUOTE_NONNUMERIC) - #### the values from the dict docs[0] can be used to find the column headers - ## in the field_list dict. + + headers = [] + for doc in sr.docs: + for field in doc: + if field not in headers: + headers.append(field) + label_header = [] - headers = list(sr.docs[0]) for colname in headers: colname = colname.strip(" _,").lower() if colname.lower() == 'id': @@ -61,23 +65,23 @@ def cache_search_results_file(cached_filename: str, sr: SolrResponse, rows=0, fi csv_file.write(u'\N{BOM}' + data) ###### @TODO - Need to remove header_file immediately - # Use a CSV writer with forced quoting for the body of the file with open(cached_filename, 'a', newline='', encoding='utf8') as csv_file: cache_writer = csv.writer(csv_file, dialect='excel', quoting=csv.QUOTE_ALL) c = 0 - for i in sr.docs: - if rows != 0 and c > rows: + for doc in sr.docs: + if rows and c >= rows: break - try: - for j in i.keys(): - if isinstance(i[j], list): - i[j] = ",".join(i[j]) - cache_writer.writerow(i.values()) - c += 1 - except UnicodeEncodeError: - pass + + values = [] + for colname in headers: + value = doc.get(colname, "") + if isinstance(value, list): + value = ",".join(str(v) for v in value) + values.append(value) + cache_writer.writerow(values) + c += 1 return True ## OPEN-4055: Accept a list of Fields for the exported search results