changes

conf/configuration.yml

@@ -9,7 +9,7 @@ SERVICE_CONFIG:
 # for testing use temp/mnt/ilens/ai-jobs/
 #--------------System Configurations--------------------#
 SYSTEM_CONFIG:
-  shared_mount_base_ai_job: temp/mnt/ilens/ai-jobs/
+  shared_mount_base_ai_job: /mnt/ilens/ai-jobs/
   default_run_id: default_run_id
 
 #----------------------If read conf from mongo------------#

main.py

@@ -50,7 +50,7 @@ class MergeWeatherData:
             logger.info("Creating X train and Y train ..............")
             x_train_df = data_train_df.drop(columns=["value"])
             y_train_df = data_train_df[["value"]]
-            x_train_df.to_csv(os.path.join(self.component_output_dir,"X_train.csv"))
+            x_train_df.to_csv(os.path.join(self.component_output_dir, "X_train.csv"))
             y_train_df.to_csv(os.path.join(self.component_output_dir, "Y_train.csv"))
             return True
         except Exception as e:
@@ -62,9 +62,11 @@ class MergeWeatherData:
             logger.info("Cleaning the historical data ..............")
             hist_df.reset_index(inplace=True)
             hist_df["Date"] = pd.to_datetime(hist_df["Date time"])
+            # print(hist_df.columns)
             hist_df.drop(
-                columns=["index", "Wind Chill", "Wind Gust", "Name", "Date time", "Snow Depth", "Snow", "Conditions",
-                         "Heat Index"], inplace=True)
+                columns=["index", "Wind Chill", "Wind Gust", "Name", "Date time", "Snow Depth", "Conditions",
+                         "Heat Index", "Address", "Dew Point", "Sea Level Pressure", "Weather Type", "Longitude",
+                         "Latitude", "Resolved Address", "Info","Precipitation Cover"], inplace=True)
             return hist_df
         except Exception as e:
             logger.info(traceback.format_exc())

scripts/common/config_parser.py

@@ -18,10 +18,7 @@ else:
     sys.stderr.write("Exiting....")
     sys.exit(1)
 # uncomment for Testing
-os.environ['pipeline_id'] = "pipe1"
-# os.environ['artifact_base_path'] = "/data/ai-models"
-# os.environ['ilens_tag_hierarchy'] = 'site1'
-# os.environ['container_name'] = 'test'
+# os.environ['pipeline_id'] = "pipe1"
 # ------------------------ Configurations -----------------------------------------------------------------------------
 pipeline_id = os.environ.get('PIPELINE_ID', default="pipeline_313")
 shared_mount_base_ai_job = os.environ.get("shared_mount_base_ai_job",

temp/mnt/ilens/ai-jobs/pipe1/default_run_id/MergeWeatherData/param.json

 {
-  "historic_file_name": "data1.csv"
+  "historic_file_name": "ahmedabad.csv"
 }
\ No newline at end of file

temp/mnt/ilens/ai-jobs/pipe1/default_run_id/ModelFitting/input/Y_train.csv

 ,value
-0,553000
-1,834000
-2,626000
-3,961000
-4,976000
-5,913000
-6,921000
-7,934000
-8,999000
-9,988000
-10,955000
-11,858000
-12,934000
-13,891000
-14,680000
-15,957000
-16,961000
-17,920000
-18,957000
-19,817000
-20,486000
-21,928000
-22,458000
-23,542000
-24,778000
-25,536000
-26,708000
-27,706000
-28,763000
-29,742000
-30,662000
+0,990000
+1,540000
+2,940000
+3,730000
+4,770000
+5,820000
+6,800000
+7,510000
+8,880000
+9,810000
+10,770000
+11,810000
+12,750000
+13,450000
+14,800000
+15,760000
+16,840000
+17,810000
+18,740000
+19,0
+20,520000
+21,0
+22,0
+23,850000
+24,1950000
+25,890000
+26,890000
+27,820000
+28,800000
+29,490000
+30,140000
+31,410000
+32,1040000
+33,970000
+34,1020000
+35,720000
+36,920000
+37,850000
+38,0
+39,0
+40,1000000
+41,350000
+42,420000
+43,600000
+44,750000
+45,420000
+46,240000
+47,850000
+48,2040000
+49,930000
+50,950000
+51,840000
+52,600000
+53,390000
+54,960000
+55,230000
+56,0
+57,290000
+58,680000
+59,500000
+60,590000
+61,810000
+62,350000
+63,420000
+64,2090000
+65,950000
+66,2090000
+67,30000
+68,720000
+69,840000
+70,0
+71,0
+72,580000
+73,670000
+74,1680000
+75,810000
+76,0
+77,790000
+78,550000
+79,300000
+80,690000
+81,590000
+82,690000
+83,420000
+84,750000
+85,850000
+86,810000
+87,460000
+88,830000
+89,580000
+90,950000
+91,620000
+92,460000
+93,820000
+94,730000
+95,800000
+96,820000
+97,520000
+98,920000
+99,950000
+100,640000
+101,1020000
+102,1070000
+103,740000
+104,670000
+105,520000
+106,230000
+107,520000
+108,560000
+109,330000
+110,580000
+111,800000
+112,870000
+113,440000
+114,610000
+115,840000
+116,870000
+117,720000
+118,940000
+119,970000
+120,740000
+121,600000
+122,590000
+123,270000
+124,190000
+125,350000
+126,50000
+127,150000
+128,150000
+129,690000
+130,600000
+131,290000
+132,370000
+133,890000
+134,340000
+135,490000
+136,760000
+137,670000
+138,720000
+139,400000
+140,650000
+141,810000
+142,740000
+143,880000
+144,810000
+145,750000
+146,620000
+147,560000
+148,690000
+149,520000
+150,720000
+151,700000
+152,560000
+153,810000
+154,330000
+155,790000
+156,820000
+157,960000
+158,870000
+159,610000
+160,620000
+161,550000
+162,820000
+163,890000
+164,860000
+165,760000
+166,810000
+167,500000
+168,350000
+169,700000
+170,890000
+171,1110000
+172,100000
+173,100000
+174,829000
+175,896000
+176,750000
+177,852000
+178,986000
+179,929000
+180,1163000
+181,1039000
+182,1091000
+183,1177000
+184,841000
+185,760000
+186,839000
+187,1141000
+188,1188000
+189,767000
+190,188000
+191,813000
+192,1032000
+193,631000
+194,1139000
+195,500000
+196,796000
+197,760000
+198,458000
+199,528000
+200,324000
+201,1186000
+202,1162000
+203,998000
+204,1067000
+205,1017000
+206,821000
+207,1000000
+208,653000
+209,1029000
+210,1003000
+211,957000
+212,687000
+213,877000
+214,1065000
+215,1020000
+216,835000
+217,895000
+218,947000
+219,827000
+220,975000
+221,1044000
+222,990000
+223,979000
+224,1029000
+225,807000
+226,983000
+227,927000
+228,832000
+229,969000
+230,611000
+231,553000
+232,834000
+233,626000
+234,961000
+235,976000
+236,913000
+237,921000
+238,934000
+239,999000
+240,988000
+241,955000
+242,858000
+243,934000
+244,891000
+245,680000
+246,957000
+247,961000
+248,920000
+249,957000
+250,817000
+251,486000
+252,928000
+253,458000
+254,542000
+255,778000
+256,536000
+257,708000
+258,706000
+259,763000
+260,742000
+261,662000
+262,660000
+263,698000
+264,757000
+265,723000
+266,665000
+267,745000
+268,750000
+269,750000
+270,732000
+271,697000
+272,803000
+273,774000
+274,655000
+275,640000
+276,694000
+277,677000
+278,689000
+279,537000
+280,675000
+281,785000
+282,781000
+283,815000
+284,786000
+285,760000
+286,797000
+287,789000
+288,739000
+289,689000
+290,737000
+291,746000
+292,737000
+293,725000
+294,322000
+295,770000
+296,771000
+297,738000
+298,631000
+299,397000
+300,689000
+301,751000
+302,766000
+303,793000
+304,791000
+305,758000
+306,712000
+307,665000
+308,687000
+309,782000
+310,806000
+311,667000
+312,816000
+313,800000
+314,747000
+315,787000
+316,769000
+317,744000
+318,713000
+319,615000
+320,734000
+321,685000
+322,775000
+323,611000
+324,644000
+325,761000
+326,340000
+327,866000
+328,804000
+329,791000
+330,816000
+331,730000
+332,620000
+333,798000
+334,934000
+335,952000
+336,915000
+337,890000
+338,877000
+339,931000
+340,909000
+341,876000
+342,864000
+343,798000
+344,694000
+345,818000
+346,461000
+347,849000
+348,874000
+349,882000
+350,843000
+351,908000
+352,931000
+353,970000
+354,816000
+355,866000
+356,877000
+357,619000
+358,874000
+359,286000
+360,849000
+361,887000
+362,1002000
+363,895000
+364,954000
+365,884000
+366,978000
+367,972000
+368,963000
+369,914000
+370,558000
+371,890000
+372,923000
+373,844000
+374,843000
+375,850000
+376,700000
+377,886000
+378,746000
+379,1325000
+380,931000
+381,573000
+382,928000
+383,1004000
+384,862000
+385,858000
+386,831000
+387,848000
+388,824000
+389,1003000
+390,682000
+391,805000
+392,815000
+393,710000
+394,823000
+395,917000
+396,988000
+397,917000
+398,682000
+399,738000
+400,442000
+401,387000
+402,704000
+403,764000
+404,678000
+405,743000
+406,716000
+407,741000
+408,785000
+409,619000
+410,765000
+411,817000
+412,403000
+413,943000
+414,590000
+415,359000
+416,812000
+417,793000
+418,699000
+419,249000
+420,611000
+421,848000
+422,367000
+423,547000
+424,531000
+425,310000
+426,571000
+427,460000
+428,841000
+429,306000
+430,456000
+431,418000
+432,510000
+433,772000
+434,1003000
+435,759000
+436,454000
+437,384000
+438,776000
+439,879000
+440,903000
+441,606000
+442,462000
+443,279000
+444,624000
+445,632000
+446,790000
+447,543000
+448,717000
+449,404000
+450,528000
+451,757000
+452,709000
+453,556000
+454,853000
+455,626000
+456,327000
+457,415000
+458,658000
+459,418000
+460,921000
+461,1079000
+462,918000
+463,797000
+464,724000
+465,967000
+466,480000
+467,896000
+468,693000
+469,72000
+470,497000
+471,754000
+472,569000
+473,524000
+474,509000
+475,456000
+476,698000
+477,437000
+478,170000
+479,612000
+480,345000
+481,527000
+482,624000
+483,705000
+484,755000
+485,646000
+486,767000
+487,812000
+488,834000
+489,764000
+490,792000
+491,756000
+492,780000
+493,716000
+494,328000
+495,930000
+496,951000
+497,764000
+498,948000
+499,1113000
+500,865000
+501,880000
+502,918000
+503,1029000
+504,575000
+505,996000
+506,931000
+507,913000
+508,546000
+509,647000
+510,461000
+511,480000
+512,447000
+513,308000
+514,564000
+515,455000
+516,630000
+517,892000
+518,714000
+519,661000
+520,522000
+521,270000
+522,403000
+523,335000
+524,791000
+525,976000
+526,695000
+527,410000
+528,565000
+529,892000
+530,867000
+531,524000
+532,921000
+533,1015000
+534,845000
+535,1906000