First working csv translator for csv upload to database (the upload process should be automatized for now only work trough phpmyadmin

3 years ago · 6ae8bb385e
parent c7977fe8cd
commit 6ae8bb385e
4 changed files with 101 additions and 75 deletions
--- a/databaseSetup/create_database.sql
+++ b/databaseSetup/create_database.sql
@ -18,7 +18,7 @@ CREATE TABLE products (
 	  price_change_flag tinyint(1) NOT NULL COMMENT 'did the price changed since the last update ? this flag should be carefully monitored to be sure that every client was informed before resetting it !!!',
 	  currency varchar(6) NOT NULL COMMENT 'The currency with which the manufaturer sells'
 	  lead_time_days int(11) NOT NULL COMMENT 'Actual lead time on this product in days',
-	  physical_form int(11) NOT NULL COMMENT 'The physical form of the product. is it a flower ? or is it a box or is it tssop18 packaged IC ?',
+	  physical_form varchar(50) NOT NULL COMMENT 'The physical form of the product. is it a flower ? or is it a box or is it tssop18 packaged IC ?',
 	  packing varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'Available packing options for the product, plastic wrap, tray, bulk, reel and so on.',
 	  packing_size int(11) NOT NULL COMMENT 'How many items are there in one packing ? Ex : A basket of 50 tomatoes. ',
 	  minimum_order_quantity int(11) NOT NULL COMMENT 'What is the minimum order quantity that the manufacturer allows. no one will send you only one flower seed. ',
@ -35,7 +35,7 @@ CREATE TABLE products (
 	  last_time_buy_date date NOT NULL COMMENT 'Last time that this product can be purchased',
 	  obsolete_date date NOT NULL COMMENT 'The date since/at which this product will be declared obsolete',
 	  pollution_level varchar(50) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'Depending on the Product some pollution level requirement can be met or not.',
-	  production_site int(11) NOT NULL COMMENT 'The production site for this product. This should be linked to the manufacturer profile.',
+	  production_site varchar(20) NOT NULL COMMENT 'The production site for this product. This should be linked to the manufacturer profile.',
 	  comment_manufacturer varchar(200) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'Comments fomr the manufaturer'
 	  comment varchar(200) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'Special comments. '
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
--- a/test/output.csv
+++ b/test/output.csv
@ -0,0 +1,18 @@
 product_id,manufacturer,product_no,core_product,ordering_code,description,status,buy_cost,pricing_scheme,price_change_flag,currency,lead_time_days,physical_form,packing,packing_size,minimum_order_quantity,minimum_awarding_quantity,minimum_quote_quantity,proposed_margin,container,container_size,product_class,product_category,product_family,product_group,last_update_date,last_time_buy_date,obsolete_date,pollution_level,production_site,comment_manufacturer,comment
 NULL,Infineon,111-4189PBF,,SP001603382,IR_POL,ACTIVE_AND_PREFERRED,287.1,Standard,0,USD,12,PQFN_5X7_34L,TAPE_&_REEL,250,250,0,1000000,25,none,250,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,no_quoting,Test Upload
 NULL,Infineon,111-4190PBF,,SP001603404,IR_POL,ACTIVE_AND_PREFERRED,338.8,Standard,0,USD,12,PQFN_5X7_34L,TAPE_&_REEL,250,250,0,1000000,25,none,250,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,no_quoting,Test Upload
 NULL,Infineon,1EBN1001AEXUMA1,1EBN1001,SP001002438,ISOLATED_HVGD,ACTIVE_AND_PREFERRED,124.15,DesignIn,0,USD,29,PG-DSO-14,TAPE_&_REEL,2500,2500,3473,3473,25,none,2500,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
 NULL,Infineon,1EBN1002AEXUMA1,1EBN1002,SP001425878,ISOLATED_HVGD,ACTIVE_AND_PREFERRED,121.68,DesignIn,0,USD,29,PG-DSO-14,TAPE_&_REEL,2500,2500,3544,3544,25,none,2500,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
 NULL,Infineon,1ED020I12B2XUMA1,1ED020,SP000898878,ISOLATED_DRIVER,ACTIVE_AND_PREFERRED,162.2,DesignIn,0,USD,67,PG-DSO-16,TAPE_&_REEL,1000,1000,1796,1796,25,none,1000,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
 NULL,Infineon,1ED020I12BTXUMA1,1ED020,SP000920032,ISOLATED_DRIVER,ACTIVE_AND_PREFERRED,162.2,DesignIn,0,USD,67,PG-DSO-16,TAPE_&_REEL,1000,1000,4000,4000,25,none,1000,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
 NULL,Infineon,1ED020I12FXUMA2,1ED020,SP000922444,ISOLATED_DRIVER,NOT_FOR_NEW_DESIGN,209.6,Standard,0,USD,67,PG-DSO-16,TAPE_&_REEL,1000,1000,0,3000,25,none,1000,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
 NULL,Infineon,1ED020I12F2XUMA1,1ED020,SP000920028,ISOLATED_DRIVER,ACTIVE_AND_PREFERRED,152.43,DesignIn,0,USD,67,PG-DSO-16,TAPE_&_REEL,1000,1000,1985,1985,25,none,1000,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
 NULL,Infineon,1ED020I12FA2XUMA2,1ED020,SP001080574,ISOLATED_HVGD,ACTIVE_AND_PREFERRED,250.1,DesignIn,0,USD,29,PG-DSO-20,TAPE_&_REEL,1000,1000,1724,1724,25,none,1000,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,replacement_for_SP000898880,Test Upload
 NULL,Infineon,1ED020I12FTXUMA1,1ED020,SP000920030,ISOLATED_DRIVER,ACTIVE_AND_PREFERRED,152.43,DesignIn,0,USD,67,PG-DSO-16,TAPE_&_REEL,1000,1000,4000,4000,25,none,1000,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
 NULL,Infineon,1ED020I12FTAXUMA2,1ED020,SP001054670,ISOLATED_HVGD,ACTIVE_AND_PREFERRED,266.05,DesignIn,0,USD,29,PG-DSO-20,TAPE_&_REEL,1000,1000,1620,1620,25,none,1000,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
 NULL,Infineon,1ED3120MC12HXUMA1,1ED3120,SP001878178,ISOLATED_DRIVER,ACTIVE_AND_PREFERRED,95.66,DesignIn,0,USD,67,PG-DSO-8,TAPE_&_REEL,1000,1000,4007,4007,25,none,1000,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
 NULL,Infineon,1ED3120MU12HXUMA1,1ED3120,SP005352070,ISOLATED_DRIVER,ACTIVE_AND_PREFERRED,90.08,DesignIn,0,USD,67,PG-DSO-8,TAPE_&_REEL,1000,1000,3521,3521,25,none,1000,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
 NULL,Infineon,1ED3121MC12HXUMA1,1ED3121,SP001878188,ISOLATED_DRIVER,ACTIVE_AND_PREFERRED,95.66,DesignIn,0,USD,67,PG-DSO-8,TAPE_&_REEL,1000,1000,4007,4007,25,none,1000,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
 NULL,Infineon,1ED3121MU12HXUMA1,1ED3121,SP005353276,ISOLATED_DRIVER,ACTIVE_AND_PREFERRED,90.08,DesignIn,0,USD,67,PG-DSO-8,TAPE_&_REEL,1000,1000,3521,3521,25,none,1000,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
 NULL,Infineon,1ED3122MC12HXUMA1,1ED3122,SP001878172,ISOLATED_DRIVER,ACTIVE_AND_PREFERRED,103.85,DesignIn,0,USD,67,PG-DSO-8,TAPE_&_REEL,1000,1000,3691,3691,25,none,1000,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
 NULL,Infineon,1ED3122MU12HXUMA1,1ED3122,SP005352068,ISOLATED_DRIVER,ACTIVE_AND_PREFERRED,98.27,DesignIn,0,USD,67,PG-DSO-8,TAPE_&_REEL,1000,1000,3521,3521,25,none,1000,0,0,0,0,2022-08-14,0000-00-00,0000-00-00,,DE,,Test Upload
--- a/test/parser.py
+++ b/test/parser.py
@ -1,32 +0,0 @@
 import csv
 import mariadb
 import sys
 # Connect to MariaDB Platform
 #try:
 #    conn = mariadb.connect(   
 #        user="db_user",
 #        password="db_user_passwd",
 #        host="192.0.2.1",
 #        port=3306,  
 #        database="employees"
 #    )   
 #except mariadb.Error as e:
 #    print(f"Error connecting to MariaDB Platform: {e}")
    # Get Cursor
 #    cur = conn.cursor()
 with open('lists/ifx_price_csv.csv') as csv_file:
    csv_reader = csv.reader(csv_file, delimiter=',')
    line_count = 0
    for row in csv_reader:
        if line_count == 0:
            print(f'{row[0]}')
            line_count += 1
        else:
            print(f'\t{row[0]} works in the {row[1]} department, and was born in {row[2]}.')
            line_count += 1
    print(f'Processed {line_count} lines.')
--- a/test/parser.sh
+++ b/test/parser.sh
@ -3,11 +3,18 @@
 INPUT_FILE=lists/ifx_price_csv.csv
 CSV_LINES=() 
 LINE_CNT=0
 COL_CNT=0
 DATABASE_COLS=0
 DELIMITER=','
-prducts_arr=(
+HEADER_STRING=""
 LINE_STRING=""
 PRODUCTS_ARR=(
 	  "product_id"
 	  "manufacturer" 
 	  "product_no"
 	  "core_product"
@ -40,36 +47,44 @@ prducts_arr=(
 	  "comment_manufacturer"
 	  "comment") 
-  manufacturer="Infineon" 
+product_id="NULL"
-  product_no=2
+manufacturer="Infineon" 
-  core_product=6
+product_no=2
-  ordering_code=1
+core_product=6
-  description=10
+ordering_code=1
-  status=22
+description=10
-  buy_cost=33
+status=22
-  pricing_scheme=20 
+buy_cost=33
-  price_change_flag=0 
+pricing_scheme=20 
-  lead_time_days=8
+price_change_flag=0 
-  physical_form=25
+currency="USD"
-  packing=26
+lead_time_days=8
-  packing_size=27 
+physical_form=25
-  minimum_order_quantity=29 
+packing=26
-  minimum_awarding_quantity=31 
+packing_size=27 
-  minimum_quote_quantity=30
+minimum_order_quantity=29 
-  proposed_margin=25
+minimum_awarding_quantity=31 
-  container="none"
+minimum_quote_quantity=30
-  container_size=29 
+proposed_margin=25
-  product_class=0 
+container="none"
-  product_category=0 
+container_size=29 
-  product_family=0 
+product_class=0 
-  product_group=0
+product_category=0 
-  last_update_date="2022-08-14" 
+product_family=0 
-  last_time_buy_date="" 
+product_group=0
-  obsolete_date=""
+last_update_date="2022-08-14" 
-  pollution_level=""
+last_time_buy_date="0000-00-00" 
-  production_site="DE"
+obsolete_date="0000-00-00"
-  comment_manufacturer=44 
+pollution_level=""
-  comment="Test Upload" 
+production_site="DE"
 comment_manufacturer=44 
 comment="Test Upload" 
 ####################################################################################################
 rm output.csv
 touch output.csv
 ####################################################################################################
 while IFS= read -r line 
 do
@ -77,24 +92,46 @@ do
 	((LINE_CNT++))
 done < $INPUT_FILE
 ####################################################################################################
 IFS=$DELIMITER read -ra HEADER <<< "${CSV_LINES[0]}"   
-
+####################################################################################################
-for i in "${HEADER[@]}"; do   # access each element of array
+for i in "${HEADER[@]}"; 
 do
 		((COL_CNT++))
 done
-echo "Total line Count = $LINE_CNT"
+for i in "${PRODUCTS_ARR[@]}"; 
-echo "Total col Count  = $COL_CNT"
+do 
 		((DATABASE_COLS++))
 done
 ####################################################################################################
 counter=0
 for i in "${PRODUCTS_ARR[@]}";do
 	((counter++))
 	if [ $counter -eq  $DATABASE_COLS ]
 	then
 		HEADER_STRING+="$i"
 		#HEADER_STRING+="$(echo "")"
 	else 
 		HEADER_STRING+="$i$DELIMITER"
 	fi
 done 
 echo "$HEADER_STRING" >> output.csv
 echo "Delimiter string : $HEADER_STRING"
 ####################################################################################################
 counter=0
 for i in "${CSV_LINES[@]}";do
 	IFS=$DELIMITER read -ra LINE <<< "${CSV_LINES[$counter]}"   
 if [ $counter -gt 1 ]
 then
-echo "$manufacturer,\
+LINE_STRING="$(echo "$product_id,\
 $manufacturer,\
 ${LINE[$product_no]},\
 ${LINE[$core_product]},\
 ${LINE[$ordering_code]},\
@ -102,7 +139,8 @@ ${LINE[$description]},\
 ${LINE[$status]},\
 ${LINE[$buy_cost]},\
 ${LINE[$pricing_scheme]},\
-${LINE[$price_change_flag]},\
+$price_change_flag,\
 $currency,\
 ${LINE[$lead_time_days]},\
 ${LINE[$physical_form]},\
 ${LINE[$packing]},\
@ -123,8 +161,10 @@ $obsolete_date,\
 $pollution_level,\
 $production_site,\
 ${LINE[$comment_manufacturer]},\
-$comment"
+$comment")"
 echo "$LINE_STRING"
 echo "$LINE_STRING" >> output.csv
 fi
 ((counter++))
 done
 ####################################################################################################