Found that the code generates successful import files. But they are crashing Energy dashboard.
This commit is contained in:
@@ -39,11 +39,19 @@ meta_lookup = meta_df.merge(meta_archive_df, on=['statistic_id'], how='left', in
|
||||
meta_lookup.set_index('id_x').to_csv("meta_merge.csv")
|
||||
meta_lookup = meta_lookup[['id_y','id_x']]
|
||||
meta_lookup = meta_lookup.T.to_dict('records')[0]
|
||||
meta_lookup = dict((v,k) for k,v in meta_lookup.items())
|
||||
print(meta_lookup)
|
||||
|
||||
############
|
||||
# statistics
|
||||
############
|
||||
|
||||
# correct the meta column
|
||||
statistics_archive_df.replace({'metadata_id': meta_lookup}, inplace=True)
|
||||
|
||||
# drop any statistics not in the existing systems metadata
|
||||
statistics_archive_df = statistics_archive_df[statistics_archive_df['metadata_id'].isin(meta_lookup.keys())]
|
||||
|
||||
# make unique indexes
|
||||
statistics_max_id = statistics_df.last_valid_index()
|
||||
statistics_archive_df.reset_index(inplace=True)
|
||||
@@ -67,16 +75,18 @@ statistics_archive_df = statistics_archive_df[~statistics_archive_df['unique_tup
|
||||
statistics_archive_df.drop(columns='unique_tuple',inplace=True)
|
||||
print(statistics_archive_df.info())
|
||||
|
||||
# drop any statistics not in the existing systems metadata
|
||||
statistics_archive_df = statistics_archive_df[statistics_archive_df['metadata_id'].isin(meta_lookup.keys())]
|
||||
|
||||
# correct the meta column
|
||||
statistics_archive_df.replace({'metadata_id': meta_lookup}, inplace=True)
|
||||
|
||||
#######################
|
||||
# statistics_short_term
|
||||
#######################
|
||||
|
||||
# correct the meta column
|
||||
statistics_short_term_archive_df.replace({'metadata_id': meta_lookup}, inplace=True)
|
||||
|
||||
# drop any statistics not in the existing systems metadata
|
||||
statistics_short_term_archive_df = statistics_short_term_archive_df[statistics_short_term_archive_df['metadata_id'].isin(meta_lookup.keys())]
|
||||
|
||||
# make unique indexes
|
||||
statistics_short_term_max_id = statistics_short_term_df.last_valid_index()
|
||||
statistics_short_term_archive_df.reset_index(inplace=True)
|
||||
@@ -102,11 +112,6 @@ statistics_short_term_archive_df.drop(columns='unique_tuple',inplace=True)
|
||||
print(statistics_short_term_archive_df.info())
|
||||
|
||||
|
||||
# drop any statistics not in the existing systems metadata
|
||||
statistics_short_term_archive_df = statistics_short_term_archive_df[statistics_short_term_archive_df['metadata_id'].isin(meta_lookup.keys())]
|
||||
|
||||
# correct the meta column
|
||||
statistics_short_term_archive_df.replace({'metadata_id': meta_lookup}, inplace=True)
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user