From fbdc9ca21ef0c55358ee72b96e1c7a327dd4e61e Mon Sep 17 00:00:00 2001 From: Shaun Setlock Date: Fri, 22 Oct 2021 21:14:45 -0400 Subject: [PATCH 1/2] Starting code for writing to Postgres db. --- db_tools/db_writer.py | 103 ++++++++++++++++++++++++++++++++++++++ db_tools/requirements.txt | 13 +++++ 2 files changed, 116 insertions(+) create mode 100644 db_tools/db_writer.py create mode 100644 db_tools/requirements.txt diff --git a/db_tools/db_writer.py b/db_tools/db_writer.py new file mode 100644 index 0000000..82d4633 --- /dev/null +++ b/db_tools/db_writer.py @@ -0,0 +1,103 @@ +#! env/bin/python3 + +# import the psycopg2 database adapter for PostgreSQL +from psycopg2 import connect, Error +import json +from psycopg2.extras import Json +from psycopg2.extras import json as psycop_json +import sys + +def connect_db(db: str,host: str,user: str,passwd: str): + try: + # declare a new PostgreSQL connection object + conn = connect( + dbname = db, + user = user, + host = host, + password = passwd, + # attempt to connect for 3 seconds then raise exception + connect_timeout = 3 + ) + cur = conn.cursor() + + except (Exception, Error) as err: + #print ("\npsycopg2 connect error:", err) + #conn = None + cur = None + return cur + +# use Python's open() function to load the JSON data +with open('postgres-records.json') as json_data: + + # use load() rather than loads() for JSON files + record_list = json.load(json_data) + +# concatenate an SQL string +sql_string = 'INSERT INTO {} '.format( table_name ) + +# if record list then get column names from first key +if type(record_list) == list: + first_record = record_list[0] + + columns = list(first_record.keys()) + print ("\ncolumn names:", columns) + +# if just one dict obj or nested JSON dict +else: + print ("Needs to be an array of JSON objects") + sys.exit() + +# enclose the column names within parenthesis +sql_string += "(" + ', '.join(columns) + ")\nVALUES " + +# only attempt to execute SQL if cursor is valid +if cur != None: + + try: + cur.execute( sql_string ) + conn.commit() + + print ('\nfinished INSERT INTO execution') + + except (Exception, Error) as error: + print("\nexecute_sql() error:", error) + conn.rollback() + + # close the cursor and connection + cur.close() + conn.close() + +######################################################3 + +import psycopg2 +from config import config + + +def insert_vendor(vendor_name): + """ insert a new vendor into the vendors table """ + sql = """INSERT INTO vendors(vendor_name) + VALUES(%s) RETURNING vendor_id;""" + conn = None + vendor_id = None + try: + # read database configuration + params = config() + # connect to the PostgreSQL database + conn = psycopg2.connect(**params) + # create a new cursor + cur = conn.cursor() + # execute the INSERT statement + cur.execute(sql, (vendor_name,)) + # get the generated id back + vendor_id = cur.fetchone()[0] + # commit the changes to the database + conn.commit() + # close communication with the database + cur.close() + except (Exception, psycopg2.DatabaseError) as error: + print(error) + finally: + if conn is not None: + conn.close() + + return vendor_id \ No newline at end of file diff --git a/db_tools/requirements.txt b/db_tools/requirements.txt new file mode 100644 index 0000000..8cf3025 --- /dev/null +++ b/db_tools/requirements.txt @@ -0,0 +1,13 @@ +black==21.9b0 +click==8.0.3 +importlib-metadata==4.8.1 +mypy-extensions==0.4.3 +pathspec==0.9.0 +platformdirs==2.4.0 +psycopg2-binary==2.9.1 +regex==2021.10.8 +RPi.GPIO==0.7.0 +tomli==1.2.1 +typed-ast==1.4.3 +typing-extensions==3.10.0.2 +zipp==3.6.0 From 1322637fef1d3031e8307c02d82224526acd0d46 Mon Sep 17 00:00:00 2001 From: Shaun Setlock Date: Sat, 23 Oct 2021 07:47:59 -0400 Subject: [PATCH 2/2] Some working db writer code. --- .gitignore | 2 + db_tools/db_writer.py | 110 +++++++++++++++--------------------------- 2 files changed, 42 insertions(+), 70 deletions(-) diff --git a/.gitignore b/.gitignore index f8b73e7..69e75a6 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,5 @@ dmypy.json # Cython debug symbols cython_debug/ +# Secrets +.creds* diff --git a/db_tools/db_writer.py b/db_tools/db_writer.py index 82d4633..1da6880 100644 --- a/db_tools/db_writer.py +++ b/db_tools/db_writer.py @@ -1,16 +1,15 @@ #! env/bin/python3 # import the psycopg2 database adapter for PostgreSQL -from psycopg2 import connect, Error -import json +import psycopg2 from psycopg2.extras import Json -from psycopg2.extras import json as psycop_json +import json import sys def connect_db(db: str,host: str,user: str,passwd: str): try: # declare a new PostgreSQL connection object - conn = connect( + conn = psycopg2.connect( dbname = db, user = user, host = host, @@ -18,86 +17,57 @@ def connect_db(db: str,host: str,user: str,passwd: str): # attempt to connect for 3 seconds then raise exception connect_timeout = 3 ) - cur = conn.cursor() - except (Exception, Error) as err: + except (Exception, psycopg2.Error) as err: #print ("\npsycopg2 connect error:", err) - #conn = None - cur = None - return cur + conn = None + return conn -# use Python's open() function to load the JSON data -with open('postgres-records.json') as json_data: - - # use load() rather than loads() for JSON files - record_list = json.load(json_data) - -# concatenate an SQL string -sql_string = 'INSERT INTO {} '.format( table_name ) - -# if record list then get column names from first key -if type(record_list) == list: - first_record = record_list[0] - - columns = list(first_record.keys()) - print ("\ncolumn names:", columns) - -# if just one dict obj or nested JSON dict -else: - print ("Needs to be an array of JSON objects") - sys.exit() - -# enclose the column names within parenthesis -sql_string += "(" + ', '.join(columns) + ")\nVALUES " - -# only attempt to execute SQL if cursor is valid -if cur != None: +def get_db_creds(file: str): + with open(file) as cred_file: + creds = json.load(cred_file) + return creds +def insert_data(conn, data): + + # insert a new vendor into the vendors table + sql = """ + INSERT INTO + air(datetime, temperature, humidity) + VALUES + (%s, %s, %s) + """ + try: - cur.execute( sql_string ) - conn.commit() - - print ('\nfinished INSERT INTO execution') - - except (Exception, Error) as error: - print("\nexecute_sql() error:", error) - conn.rollback() - - # close the cursor and connection - cur.close() - conn.close() - -######################################################3 - -import psycopg2 -from config import config - - -def insert_vendor(vendor_name): - """ insert a new vendor into the vendors table """ - sql = """INSERT INTO vendors(vendor_name) - VALUES(%s) RETURNING vendor_id;""" - conn = None - vendor_id = None - try: - # read database configuration - params = config() - # connect to the PostgreSQL database - conn = psycopg2.connect(**params) - # create a new cursor + # open cursor on our db connection cur = conn.cursor() + # execute the INSERT statement - cur.execute(sql, (vendor_name,)) - # get the generated id back - vendor_id = cur.fetchone()[0] + data = (data["datetime"], data["temperature"], data["humidity"]) + cur.execute(sql,data) + # commit the changes to the database conn.commit() + # close communication with the database cur.close() + except (Exception, psycopg2.DatabaseError) as error: print(error) + finally: if conn is not None: conn.close() + +if __name__ == "__main__": + data = { + "datetime": "2021-10-23 01:58:08.205911", + "temperature": "73.4", + "humidity": "49.2" + } - return vendor_id \ No newline at end of file + creds = get_db_creds("./.creds.json") + + conn = connect_db(creds["db"], creds["host"], creds["user"], creds["passwd"]) + + insert_data(conn, data) \ No newline at end of file