diff --git a/scripts/us_usda/quickstats/golden_data/golden_observations.csv b/scripts/us_usda/quickstats/golden_data/golden_observations.csv new file mode 100644 index 0000000000..71ad404322 --- /dev/null +++ b/scripts/us_usda/quickstats/golden_data/golden_observations.csv @@ -0,0 +1,1959 @@ +"observationAbout" +"geoId/06113" +"geoId/28135" +"geoId/28139" +"geoId/21203" +"geoId/53077" +"geoId/27127" +"geoId/40037" +"geoId/55029" +"geoId/39095" +"geoId/08069" +"geoId/05133" +"geoId/49041" +"geoId/47155" +"geoId/13185" +"geoId/28087" +"geoId/06027" +"geoId/40145" +"geoId/50025" +"geoId/17115" +"geoId/37113" +"geoId/47111" +"geoId/01087" +"geoId/27061" +"geoId/39009" +"geoId/34005" +"geoId/39129" +"geoId/16001" +"geoId/37059" +"geoId/41013" +"geoId/17029" +"geoId/22083" +"geoId/38077" +"geoId/45079" +"geoId/39139" +"geoId/55103" +"geoId/17159" +"geoId/48097" +"geoId/48375" +"geoId/42105" +"geoId/13085" +"geoId/31047" +"geoId/37175" +"geoId/45037" +"geoId/26157" +"geoId/19155" +"geoId/05031" +"geoId/19171" +"geoId/49003" +"geoId/34001" +"geoId/37091" +"geoId/06005" +"geoId/13285" +"geoId/42079" +"geoId/16017" +"geoId/36063" +"geoId/13011" +"geoId/34037" +"geoId/10005" +"geoId/26117" +"geoId/05103" +"geoId/22073" +"geoId/21095" +"geoId/02122" +"geoId/06043" +"geoId/17117" +"geoId/36073" +"geoId/50019" +"geoId/27105" +"geoId/29055" +"geoId/39033" +"geoId/42039" +"geoId/55023" +"geoId/05033" +"geoId/17033" +"geoId/19047" +"geoId/20037" +"geoId/51001" +"geoId/27059" +"geoId/27021" +"geoId/29037" +"geoId/31025" +"geoId/38017" +"geoId/48067" +"geoId/18017" +"geoId/19029" +"geoId/26027" +"geoId/36123" +"geoId/37027" +"geoId/48055" +"geoId/48157" +"geoId/34009" +"geoId/18029" +"geoId/35047" +"geoId/46005" +"geoId/13051" +"geoId/37037" +"geoId/01061" +"geoId/13111" +"geoId/48147" +"geoId/48091" +"geoId/22015" +"geoId/06037" +"geoId/47023" +"geoId/42029" +"geoId/45023" +"geoId/06081" +"geoId/01033" +"geoId/26123" +"geoId/42093" +"geoId/26019" +"geoId/22007" +"geoId/05115" +"geoId/53033" +"geoId/53067" +"geoId/26059" +"geoId/47119" +"geoId/37081" +"geoId/26067" +"geoId/28039" +"geoId/27103" +"geoId/16075" +"geoId/56039" +"geoId/06025" +"geoId/51173" +"geoId/13071" +"geoId/55133" +"geoId/37061" +"geoId/12073" +"geoId/48289" +"geoId/22017" +"geoId/40015" +"geoId/08077" +"geoId/37127" +"geoId/36017" +"geoId/51165" +"geoId/37157" +"geoId/27005" +"geoId/01109" +"geoId/13231" +"geoId/17149" +"geoId/21195" +"geoId/29163" +"geoId/39131" +"geoId/28113" +"geoId/35061" +"geoId/27047" +"geoId/47131" +"geoId/39039" +"geoId/21205" +"geoId/37159" +"geoId/40009" +"geoId/37063" +"geoId/48281" +"geoId/27153" +"geoId/22109" +"geoId/28133" +"geoId/13247" +"geoId/13115" +"geoId/19067" +"geoId/18043" +"geoId/21071" +"geoId/55127" +"geoId/13205" +"geoId/08123" +"geoId/16049" +"geoId/50005" +"geoId/37015" +"geoId/48185" +"geoId/05135" +"geoId/48463" +"geoId/18037" +"geoId/21079" +"geoId/04027" +"geoId/48021" +"geoId/27003" +"geoId/47183" +"geoId/28109" +"geoId/12123" +"geoId/21217" +"geoId/48441" +"geoId/55119" +"geoId/54091" +"geoId/35057" +"geoId/39093" +"geoId/42031" +"geoId/01059" +"geoId/13119" +"geoId/17055" +"geoId/20059" +"geoId/21073" +"geoId/29071" +"geoId/36033" +"geoId/37069" +"geoId/47051" +"geoId/51067" +"geoId/42055" +"geoId/50011" +"geoId/53021" +"geoId/05047" +"geoId/18047" +"geoId/22041" +"geoId/39049" +"geoId/13279" +"geoId/45011" +"geoId/55087" +"geoId/36067" +"geoId/45075" +"geoId/26077" +"geoId/27169" +"geoId/19059" +"geoId/20041" +"geoId/26043" +"geoId/54103" +"geoId/28117" +"geoId/48273" +"geoId/26033" +"geoId/55017" +"geoId/48209" +"geoId/42133" +"geoId/45091" +"geoId/54039" +"geoId/48439" +"geoId/51015" +"geoId/26065" +"geoId/30029" +"geoId/29091" +"geoId/40111" +"geoId/48121" +"geoId/27161" +"geoId/55105" +"geoId/37047" +"geoId/55033" +"geoId/01053" +"geoId/12033" +"geoId/01017" +"geoId/48071" +"geoId/01049" +"geoId/17037" +"geoId/18033" +"geoId/47041" +"geoId/12089" +"geoId/45089" +"geoId/37019" +"geoId/51025" +"geoId/48449" +"geoId/06071" +"geoId/20133" +"geoId/27019" +"geoId/26151" +"geoId/48323" +"geoId/27147" +"geoId/06065" +"geoId/51003" +"geoId/48061" +"geoId/17125" +"geoId/21161" +"geoId/26105" +"geoId/54053" +"geoId/53045" +"geoId/22037" +"geoId/13179" +"geoId/48291" +"geoId/36007" +"geoId/05149" +"geoId/17203" +"geoId/21239" +"geoId/41011" +"geoId/51195" +"geoId/48497" +"geoId/37053" +"geoId/13095" +"geoId/20139" +"geoId/40113" +"geoId/29165" +"geoId/31141" +"geoId/17095" +"geoId/18083" +"geoId/21121" +"geoId/39083" +"geoId/47093" +"geoId/38015" +"geoId/35009" +"geoId/41015" +"geoId/12101" +"geoId/18169" +"geoId/47057" +"geoId/08075" +"geoId/17107" +"geoId/21141" +"geoId/40083" +"geoId/39091" +"geoId/05083" +"geoId/48465" +"geoId/48401" +"geoId/22121" +"geoId/48407" +"geoId/48453" +"geoId/05141" +"geoId/26159" +"geoId/05059" +"geoId/46011" +"geoId/13183" +"geoId/45041" +"geoId/49049" +"geoId/13229" +"geoId/53053" +"geoId/55093" +"geoId/28159" +"geoId/01133" +"geoId/49035" +"geoId/29069" +"geoId/48285" +"geoId/39089" +"geoId/51175" +"geoId/05051" +"geoId/06017" +"geoId/08051" +"geoId/17179" +"geoId/51185" +"geoId/37083" +"geoId/51083" +"geoId/37119" +"geoId/51117" +"geoId/06011" +"geoId/54061" +"geoId/56001" +"geoId/36001" +"geoId/37133" +"geoId/28131" +"geoId/29209" +"geoId/13237" +"geoId/18133" +"geoId/47141" +"geoId/39137" +"geoId/12107" +"geoId/54079" +"geoId/42017" +"geoId/41005" +"geoId/36065" +"geoId/55085" +"geoId/16013" +"geoId/37009" +"geoId/21009" +"geoId/51065" +"geoId/37107" +"geoId/39005" +"geoId/55003" +"geoId/17005" +"geoId/24009" +"geoId/24015" +"geoId/06077" +"geoId/50007" +"geoId/13293" +"geoId/36083" +"geoId/34029" +"geoId/26069" +"geoId/40089" +"geoId/55073" +"geoId/22039" +"geoId/48503" +"geoId/29195" +"geoId/05125" +"geoId/17165" +"geoId/20169" +"geoId/48221" +"geoId/48469" +"geoId/04005" +"geoId/45069" +"geoId/48457" +"geoId/12117" +"geoId/40133" +"geoId/47073" +"geoId/24033" +"geoId/16069" +"geoId/08029" +"geoId/26041" +"geoId/06085" +"geoId/05027" +"geoId/12023" +"geoId/36021" +"geoId/41009" +"geoId/42037" +"geoId/55021" +"geoId/13073" +"geoId/18183" +"geoId/21235" +"geoId/13227" +"geoId/01107" +"geoId/45077" +"geoId/36043" +"geoId/55121" +"geoId/35005" +"geoId/45033" +"geoId/28035" +"geoId/01119" +"geoId/12119" +"geoId/13261" +"geoId/45085" +"geoId/21147" +"geoId/37033" +"geoId/31067" +"geoId/20005" +"geoId/13275" +"geoId/36111" +"geoId/41027" +"geoId/39133" +"geoId/55097" +"geoId/17045" +"geoId/42063" +"geoId/16053" +"geoId/20061" +"geoId/13013" +"geoId/21163" +"geoId/46093" +"geoId/01095" +"geoId/18099" +"geoId/28093" +"geoId/40095" +"geoId/47117" +"geoId/54051" +"geoId/19127" +"geoId/21157" +"geoId/42121" +"geoId/53047" +"geoId/39135" +"geoId/06035" +"geoId/47043" +"geoId/30093" +"geoId/17111" +"geoId/05023" +"geoId/12055" +"geoId/48145" +"geoId/16067" +"geoId/48451" +"geoId/54027" +"geoId/36075" +"geoId/18141" +"geoId/37021" +"geoId/13321" +"geoId/40101" +"geoId/27037" +"geoId/31043" +"geoId/54077" +"geoId/13059" +"geoId/28023" +"geoId/01025" +"geoId/37035" +"geoId/24029" +"geoId/26081" +"geoId/10001" +"geoId/21179" +"geoId/53039" +"geoId/05037" +"geoId/47049" +"geoId/17109" +"geoId/55135" +"geoId/29009" +"geoId/26015" +"geoId/48013" +"geoId/21145" +"geoId/35001" +"geoId/38035" +"geoId/26103" +"geoId/35015" +"geoId/05043" +"geoId/51085" +"geoId/26125" +"geoId/06009" +"geoId/41035" +"geoId/47045" +"geoId/21013" +"geoId/48027" +"geoId/12129" +"geoId/37141" +"geoId/48459" +"geoId/54097" +"geoId/41039" +"geoId/55063" +"geoId/18087" +"geoId/37077" +"geoId/39037" +"geoId/21049" +"geoId/53011" +"geoId/55019" +"geoId/05019" +"geoId/17023" +"geoId/18019" +"geoId/32003" +"geoId/39023" +"geoId/54059" +"geoId/53015" +"geoId/40031" +"geoId/48189" +"geoId/01065" +"geoId/26011" +"geoId/08045" +"geoId/40047" +"geoId/12115" +"geoId/48299" +"geoId/21137" +"geoId/22061" +"geoId/28085" +"geoId/29113" +"geoId/31111" +"geoId/35027" +"geoId/37109" +"geoId/40081" +"geoId/41041" +"geoId/47103" +"geoId/55069" +"geoId/30053" +"geoId/46083" +"geoId/54043" +"geoId/56023" +"geoId/48265" +"geoId/28057" +"geoId/17187" +"geoId/21227" +"geoId/28149" +"geoId/47177" +"geoId/51187" +"geoId/34041" +"geoId/42123" +"geoId/19181" +"geoId/29219" +"geoId/37185" +"geoId/39165" +"geoId/28079" +"geoId/26057" +"geoId/27111" +"geoId/45083" +"geoId/01021" +"geoId/46135" +"geoId/17083" +"geoId/54067" +"geoId/35049" +"geoId/29207" +"geoId/28003" +"geoId/37003" +"geoId/48321" +"geoId/05077" +"geoId/12071" +"geoId/17103" +"geoId/19111" +"geoId/28081" +"geoId/37105" +"geoId/45061" +"geoId/51105" +"geoId/01081" +"geoId/13177" +"geoId/48287" +"geoId/34025" +"geoId/19013" +"geoId/26165" +"geoId/26005" +"geoId/06015" +"geoId/27137" +"geoId/39029" +"geoId/55039" +"geoId/29007" +"geoId/13077" +"geoId/53003" +"geoId/38059" +"geoId/55067" +"geoId/13213" +"geoId/06033" +"geoId/12069" +"geoId/18089" +"geoId/30047" +"geoId/39085" +"geoId/17097" +"geoId/29105" +"geoId/27099" +"geoId/13157" +"geoId/17077" +"geoId/19097" +"geoId/22049" +"geoId/28059" +"geoId/29095" +"geoId/37099" +"geoId/40065" +"geoId/41029" +"geoId/26075" +"geoId/39079" +"geoId/54035" +"geoId/55053" +"geoId/01071" +"geoId/05067" +"geoId/12063" +"geoId/18071" +"geoId/17167" +"geoId/06061" +"geoId/48051" +"geoId/05021" +"geoId/12019" +"geoId/18021" +"geoId/19041" +"geoId/27027" +"geoId/28025" +"geoId/29047" +"geoId/21051" +"geoId/48489" +"geoId/39117" +"geoId/51171" +"geoId/24041" +"geoId/51143" +"geoId/01115" +"geoId/48381" +"geoId/34033" +"geoId/05093" +"geoId/12061" +"geoId/19169" +"geoId/12121" +"geoId/05071" +"geoId/18081" +"geoId/19103" +"geoId/21115" +"geoId/29101" +"geoId/47091" +"geoId/48251" +"geoId/20091" +"geoId/22019" +"geoId/48037" +"geoId/13171" +"geoId/28073" +"geoId/48277" +"geoId/40143" +"geoId/06111" +"geoId/21125" +"geoId/21183" +"geoId/54069" +"geoId/48041" +"geoId/06099" +"geoId/37101" +"geoId/05007" +"geoId/19011" +"geoId/27009" +"geoId/29015" +"geoId/41003" +"geoId/53005" +"geoId/47021" +"geoId/18045" +"geoId/27083" +"geoId/20111" +"geoId/32019" +"geoId/06107" +"geoId/24023" +"geoId/21117" +"geoId/20173" +"geoId/16031" +"geoId/18113" +"geoId/55071" +"geoId/16019" +"geoId/36077" +"geoId/26137" +"geoId/49011" +"geoId/45027" +"geoId/48397" +"geoId/05057" +"geoId/41051" +"geoId/28107" +"geoId/48365" +"geoId/20113" +"geoId/55115" +"geoId/06073" +"geoId/18173" +"geoId/17007" +"geoId/18011" +"geoId/05009" +"geoId/19015" +"geoId/21015" +"geoId/29019" +"geoId/29213" +"geoId/13123" +"geoId/12093" +"geoId/01031" +"geoId/13069" +"geoId/47031" +"geoId/30041" +"geoId/48217" +"geoId/46029" +"geoId/51197" +"geoId/29027" +"geoId/22093" +"geoId/42075" +"geoId/26161" +"geoId/54025" +"geoId/30111" +"geoId/08015" +"geoId/08117" +"geoId/39153" +"geoId/49043" +"geoId/20099" +"geoId/39145" +"geoId/40039" +"geoId/19123" +"geoId/55075" +"geoId/19033" +"geoId/13103" +"geoId/17049" +"geoId/04025" +"geoId/13029" +"geoId/40013" +"geoId/51095" +"geoId/21005" +"geoId/45007" +"geoId/47001" +"geoId/48001" +"geoId/55005" +"geoId/13241" +"geoId/22087" +"geoId/47147" +"geoId/48395" +"geoId/55117" +"geoId/04015" +"geoId/53073" +"geoId/29161" +"geoId/06031" +"geoId/45001" +"geoId/48025" +"geoId/48035" +"geoId/53037" +"geoId/01127" +"geoId/13295" +"geoId/48471" +"geoId/37013" +"geoId/45013" +"geoId/12097" +"geoId/26133" +"geoId/13033" +"geoId/37023" +"geoId/53035" +"geoId/12009" +"geoId/19191" +"geoId/36029" +"geoId/42049" +"geoId/39043" +"geoId/21113" +"geoId/42021" +"geoId/13087" +"geoId/18031" +"geoId/29189" +"geoId/48473" +"geoId/37131" +"geoId/42095" +"geoId/22001" +"geoId/01091" +"geoId/27157" +"geoId/26155" +"geoId/20155" +"geoId/37195" +"geoId/47189" +"geoId/48493" +"geoId/42007" +"geoId/51029" +"geoId/39073" +"geoId/37171" +"geoId/21101" +"geoId/37089" +"geoId/47077" +"geoId/48213" +"geoId/18143" +"geoId/21209" +"geoId/27139" +"geoId/28123" +"geoId/47151" +"geoId/51169" +"geoId/19163" +"geoId/29201" +"geoId/18003" +"geoId/21003" +"geoId/22003" +"geoId/39003" +"geoId/01093" +"geoId/05089" +"geoId/12083" +"geoId/19125" +"geoId/21155" +"geoId/29127" +"geoId/41047" +"geoId/45067" +"geoId/39101" +"geoId/17121" +"geoId/18097" +"geoId/28091" +"geoId/47115" +"geoId/54049" +"geoId/47171" +"geoId/47105" +"geoId/39007" +"geoId/27055" +"geoId/48225" +"geoId/01069" +"geoId/13153" +"geoId/55109" +"geoId/50015" +"geoId/50021" +"geoId/05063" +"geoId/18063" +"geoId/42015" +"geoId/12007" +"geoId/21027" +"geoId/35043" +"geoId/45045" +"geoId/26047" +"geoId/26145" +"geoId/37189" +"geoId/01083" +"geoId/48293" +"geoId/18005" +"geoId/20009" +"geoId/40017" +"geoId/05045" +"geoId/12086" +"geoId/39143" +"geoId/20149" +"geoId/40125" +"geoId/18079" +"geoId/51071" +"geoId/47055" +"geoId/48367" +"geoId/27145" +"geoId/27017" +"geoId/39119" +"geoId/41065" +"geoId/51177" +"geoId/01123" +"geoId/13081" +"geoId/26035" +"geoId/19083" +"geoId/21093" +"geoId/47071" +"geoId/39065" +"geoId/48199" +"geoId/27015" +"geoId/39015" +"geoId/55009" +"geoId/46013" +"geoId/48049" +"geoId/19193" +"geoId/06063" +"geoId/21085" +"geoId/48181" +"geoId/51077" +"geoId/26073" +"geoId/18059" +"geoId/28045" +"geoId/39063" +"geoId/17067" +"geoId/54029" +"geoId/48187" +"geoId/20057" +"geoId/12075" +"geoId/48349" +"geoId/28007" +"geoId/45043" +"geoId/13137" +"geoId/19105" +"geoId/48253" +"geoId/13169" +"geoId/28067" +"geoId/56033" +"geoId/26007" +"geoId/51800" +"geoId/13075" +"geoId/17031" +"geoId/38089" +"geoId/39151" +"geoId/17177" +"geoId/48479" +"geoId/12085" +"geoId/27091" +"geoId/37117" +"geoId/40091" +"geoId/37049" +"geoId/45049" +"geoId/22103" +"geoId/55083" +"geoId/13121" +"geoId/17057" +"geoId/18049" +"geoId/36035" +"geoId/39051" +"geoId/13021" +"geoId/01007" +"geoId/19017" +"geoId/13019" +"geoId/26021" +"geoId/54009" +"geoId/08037" +"geoId/29215" +"geoId/40139" +"geoId/42061" +"geoId/26127" +"geoId/51087" +"geoId/54011" +"geoId/28029" +"geoId/06087" +"geoId/04023" +"geoId/51153" +"geoId/21133" +"geoId/45059" +"geoId/13175" +"geoId/01079" +"geoId/17101" +"geoId/18093" +"geoId/29109" +"geoId/46081" +"geoId/47099" +"geoId/39087" +"geoId/42073" +"geoId/05075" +"geoId/37031" +"geoId/01019" +"geoId/20021" +"geoId/40021" +"geoId/45021" +"geoId/13057" +"geoId/19035" +"geoId/37039" +"geoId/48073" +"geoId/51033" +"geoId/24011" +"geoId/08005" +"geoId/48233" +"geoId/04007" +"geoId/24021" +"geoId/51069" +"geoId/16011" +"geoId/13219" +"geoId/45073" +"geoId/39045" +"geoId/45039" +"geoId/12029" +"geoId/01067" +"geoId/13151" +"geoId/19087" +"geoId/29083" +"geoId/47079" +"geoId/39069" +"geoId/17073" +"geoId/18065" +"geoId/51089" +"geoId/37097" +"geoId/37065" +"geoId/55059" +"geoId/13217" +"geoId/28101" +"geoId/29145" +"geoId/47069" +"geoId/12039" +"geoId/22053" +"geoId/37147" +"geoId/13277" +"geoId/42115" +"geoId/24003" +"geoId/19019" +"geoId/29021" +"geoId/51027" +"geoId/55089" +"geoId/37007" +"geoId/13311" +"geoId/18181" +"geoId/47185" +"geoId/05145" +"geoId/17193" +"geoId/17091" +"geoId/10003" +"geoId/48089" +"geoId/17131" +"geoId/21167" +"geoId/39107" +"geoId/42085" +"geoId/34021" +"geoId/54055" +"geoId/24017" +"geoId/17201" +"geoId/55139" +"geoId/28043" +"geoId/37181" +"geoId/08039" +"geoId/13105" +"geoId/19061" +"geoId/39175" +"geoId/37071" +"geoId/22011" +"geoId/29053" +"geoId/26063" +"geoId/39077" +"geoId/48215" +"geoId/37087" +"geoId/47075" +"geoId/27067" +"geoId/13047" +"geoId/42047" +"geoId/37163" +"geoId/13189" +"geoId/51149" +"geoId/18149" +"geoId/16027" +"geoId/36121" +"geoId/42131" +"geoId/18103" +"geoId/20121" +"geoId/39109" +"geoId/55049" +"geoId/39171" +"geoId/38105" +"geoId/13031" +"geoId/05055" +"geoId/17061" +"geoId/18055" +"geoId/29077" +"geoId/37079" +"geoId/47059" +"geoId/51079" +"geoId/39057" +"geoId/13133" +"geoId/36039" +"geoId/42059" +"geoId/36011" +"geoId/29143" +"geoId/22127" +"geoId/06019" +"geoId/31153" +"geoId/40049" +"geoId/06115" +"geoId/12049" +"geoId/18123" +"geoId/29157" +"geoId/39127" +"geoId/42099" +"geoId/17145" +"geoId/21193" +"geoId/05111" +"geoId/48227" +"geoId/24027" +"geoId/18067" +"geoId/29051" +"geoId/47029" +"geoId/20079" +"geoId/28163" +"geoId/48163" +"geoId/16005" +"geoId/39011" +"geoId/13207" +"geoId/18105" +"geoId/36055" +"geoId/47123" +"geoId/39111" +"geoId/55081" +"geoId/01099" +"geoId/17133" +"geoId/26115" +"geoId/28095" +"geoId/42089" +"geoId/51127" +"geoId/28011" +"geoId/51023" +"geoId/39155" +"geoId/20161" +"geoId/18085" +"geoId/13107" +"geoId/13067" +"geoId/35006" +"geoId/13017" +"geoId/22033" +"geoId/45009" +"geoId/05029" +"geoId/24045" +"geoId/42003" +"geoId/42027" +"geoId/06001" +"geoId/54081" +"geoId/08119" +"geoId/18051" +"geoId/47053" +"geoId/48309" +"geoId/45025" +"geoId/51041" +"geoId/48329" +"geoId/26111" +"geoId/51145" +"geoId/56021" +"geoId/26009" +"geoId/12041" +"geoId/49057" +"geoId/51161" +"geoId/06103" +"geoId/08041" +"geoId/48141" +"geoId/47145" +"geoId/48427" +"geoId/12001" +"geoId/26079" +"geoId/17021" +"geoId/21047" +"geoId/29043" +"geoId/05087" +"geoId/17119" +"geoId/18095" +"geoId/19121" +"geoId/21151" +"geoId/28089" +"geoId/36053" +"geoId/37115" +"geoId/39097" +"geoId/01089" +"geoId/12079" +"geoId/13195" +"geoId/16065" +"geoId/31119" +"geoId/47113" +"geoId/16055" +"geoId/39157" +"geoId/41033" +"geoId/08083" +"geoId/55111" +"geoId/12027" +"geoId/28033" +"geoId/22031" +"geoId/54057" +"geoId/13135" +"geoId/19145" +"geoId/51139" +"geoId/36103" +"geoId/18121" +"geoId/04003" +"geoId/26121" +"geoId/22079" +"geoId/36119" +"geoId/06101" +"geoId/05085" +"geoId/13015" +"geoId/17191" +"geoId/18177" +"geoId/21231" +"geoId/28153" +"geoId/36117" +"geoId/47181" +"geoId/39169" +"geoId/42127" +"geoId/13305" +"geoId/26163" +"geoId/37191" +"geoId/54099" +"geoId/36031" +"geoId/48415" +"geoId/04012" +"geoId/47153" +"geoId/51009" +"geoId/37197" +"geoId/36003" +"geoId/24001" +"geoId/28141" +"geoId/26091" +"geoId/12131" +"geoId/13297" +"geoId/31159" +"geoId/20175" +"geoId/37199" +"geoId/55047" +"geoId/51099" +"geoId/13113" +"geoId/17051" +"geoId/19065" +"geoId/21067" +"geoId/47047" +"geoId/48149" +"geoId/39047" +"geoId/42051" +"geoId/01057" +"geoId/18041" +"geoId/54019" +"geoId/35053" +"geoId/51019" +"geoId/42009" +"geoId/47003" +"geoId/40121" +"geoId/16083" +"geoId/50001" +"geoId/36091" +"geoId/48219" +"geoId/48347" +"geoId/13117" +"geoId/37067" +"geoId/53075" +"geoId/17143" +"geoId/06069" +"geoId/42077" +"geoId/48171" +"geoId/27007" +"geoId/50003" +"geoId/47063" +"geoId/17001" +"geoId/18001" +"geoId/28001" +"geoId/39001" +"geoId/42001" +"geoId/53001" +"geoId/08001" +"geoId/31001" +"geoId/55001" +"geoId/37183" +"geoId/19167" +"geoId/39099" +"geoId/17089" +"geoId/48165" +"geoId/37045" +"geoId/40027" +"geoId/48053" +"geoId/22069" +"geoId/49047" +"geoId/30049" +"geoId/13225" +"geoId/06023" +"geoId/32013" +"geoId/05143" +"geoId/18175" +"geoId/19183" +"geoId/22117" +"geoId/27163" +"geoId/28151" +"geoId/29221" +"geoId/31177" +"geoId/36115" +"geoId/40147" +"geoId/47179" +"geoId/49053" +"geoId/51191" +"geoId/24043" +"geoId/39167" +"geoId/42125" +"geoId/50023" +"geoId/55131" +"geoId/01129" +"geoId/12133" +"geoId/13303" +"geoId/41067" +"geoId/48477" +"geoId/39059" +"geoId/21177" +"geoId/26089" +"geoId/06039" +"geoId/20051" +"geoId/48139" +"geoId/01023" +"geoId/40023" +"geoId/55057" +"geoId/40131" +"geoId/36109" +"geoId/39055" +"geoId/16057" +"geoId/36013" +"geoId/27043" +"geoId/17085" +"geoId/48249" +"geoId/06013" +"geoId/46065" +"geoId/13145" +"geoId/48201" +"geoId/01045" +"geoId/18061" +"geoId/19085" +"geoId/21097" +"geoId/48203" +"geoId/39067" +"geoId/54033" +"geoId/28047" +"geoId/47081" +"geoId/18139" +"geoId/42097" +"geoId/48303" +"geoId/48409" +"geoId/13039" +"geoId/29029" +"geoId/01039" +"geoId/28031" +"geoId/20191" +"geoId/47165" +"geoId/53061" +"geoId/24025" +"geoId/51061" +"geoId/01073" +"geoId/05069" +"geoId/16051" +"geoId/17081" +"geoId/19101" +"geoId/22051" +"geoId/29099" +"geoId/36045" +"geoId/47089" +"geoId/48245" +"geoId/42065" +"geoId/54037" +"geoId/55055" +"geoId/08059" +"geoId/13163" +"geoId/18077" +"geoId/20087" +"geoId/21111" +"geoId/39081" +"geoId/41031" +"geoId/53031" +"geoId/19141" +"geoId/22057" +"geoId/26045" +"geoId/01005" +"geoId/30013" +"geoId/17163" +"geoId/26147" +"geoId/08097" +"geoId/13199" +"geoId/39053" +"geoId/51053" +"geoId/26031" +"geoId/28137" +"geoId/05107" +"geoId/29155" +"geoId/13313" +"geoId/28017" +"geoId/18075" +"geoId/18127" +"geoId/26029" +"geoId/35041" +"geoId/29177" +"geoId/30063" +"geoId/30031" +"geoId/28121" +"geoId/53027" +"geoId/13035" +"geoId/13025" +"geoId/35055" +"geoId/36009" +"geoId/22105" +"geoId/05131" +"geoId/06109" +"geoId/19197" +"geoId/27171" +"geoId/29229" +"geoId/21029" +"geoId/01013" +"geoId/19023" +"geoId/20015" +"geoId/42019" +"geoId/29023" +"geoId/39017" +"geoId/06045" +"geoId/05121" +"geoId/17157" +"geoId/18135" +"geoId/29175" +"geoId/37151" +"geoId/01111" +"geoId/54083" +"geoId/36105" +"geoId/18153" +"geoId/47163" +"geoId/47133" +"geoId/47107" +"geoId/17099" +"geoId/17027" +"geoId/18023" +"geoId/19045" +"geoId/36019" +"geoId/26037" +"geoId/39027" +"geoId/42035" +"geoId/29049" +"geoId/17195" +"geoId/06057" +"geoId/49021" +"geoId/31019" +"geoId/41071" +"geoId/27079" +"geoId/18069" +"geoId/29013" +"geoId/28105" +"geoId/46099" +"geoId/28099" +"geoId/34011" +"geoId/42041" +"geoId/37051" +"geoId/47035" +"geoId/36093" +"geoId/12091" +"geoId/19179" +"geoId/26023" +"geoId/18137" +"geoId/01101" +"geoId/20125" +"geoId/36057" +"geoId/47125" +"geoId/48339" +"geoId/51121" +"geoId/24031" +"geoId/17135" +"geoId/18107" +"geoId/21173" +"geoId/37123" +"geoId/39113" +"geoId/42091" +"geoId/47167" +"geoId/42107" +"geoId/17197" +"geoId/42033" +"geoId/18119" +"geoId/13001" +"geoId/48015" +"geoId/36037" +"geoId/26049" +"geoId/05053" +"geoId/21081" +"geoId/35017" +"geoId/53025" +"geoId/55043" +"geoId/18053" +"geoId/22043" +"geoId/27095" +"geoId/38101" +"geoId/08067" +"geoId/04001" +"geoId/41059" +"geoId/45051" +"geoId/20103" +"geoId/22027" +"geoId/47025" +"geoId/22029" +"geoId/49013" +"geoId/45081" +"geoId/47109" +"geoId/22115" +"geoId/55123" +"geoId/29217" +"geoId/31109" +"geoId/45057" +"geoId/42071" +"geoId/48353" +"geoId/39013" +"geoId/40149" +"geoId/48161" +"geoId/42081" +"geoId/38093" +"geoId/48029" +"geoId/40071" +"geoId/08107" +"geoId/31157" +"geoId/42111" +"geoId/34035" +"geoId/24039" +"geoId/18027" +"geoId/21059" +"geoId/27045" +"geoId/12113" +"geoId/18147" +"geoId/21215" +"geoId/17183" +"geoId/22113" +"geoId/21107" +"geoId/48223" +"geoId/42067" +"geoId/51163" +"geoId/26087" +"geoId/01009" +"geoId/47009" +"geoId/13267" +"geoId/32031" +"geoId/12015" +"geoId/36107" +"geoId/42117" +"geoId/12031" +"geoId/18151" +"geoId/36101" +"geoId/49051" +"geoId/06053" +"geoId/17093" +"geoId/48259" +"geoId/37125" +"geoId/48341" +"geoId/49007" +"geoId/56007" +"geoId/42025" +"geoId/17113" +"geoId/51107" +"geoId/40135" +"geoId/51147" +"geoId/06079" +"geoId/55035" +"geoId/48485" +"geoId/17063" +"geoId/29147" +"geoId/01097" +"geoId/17161" +"geoId/19157" +"geoId/39161" +"geoId/55045" +"geoId/48123" +"geoId/17039" +"geoId/26055" +"geoId/18035" +"geoId/19055" +"geoId/36025" +"geoId/40041" +"geoId/39041" +"geoId/37169" +"geoId/40153" +"geoId/48279" +"geoId/53009" +"geoId/26061" +"geoId/05015" +"geoId/17015" +"geoId/18015" +"geoId/19027" +"geoId/51035" +"geoId/24013" +"geoId/39019" +"geoId/13045" +"geoId/47017" +"geoId/36015" +"geoId/29003" +"geoId/35031" +"geoId/48371" +"geoId/17173" +"geoId/18145" +"geoId/21211" +"geoId/48419" +"geoId/39149" +"geoId/01117" +"geoId/45029" +"geoId/48135" +"geoId/41057" +"geoId/05119" +"geoId/21199" +"geoId/29169" +"geoId/51155" +"geoId/49045" +"geoId/40109" +"geoId/21001" +"geoId/29001" +"geoId/40001" +"geoId/21207" +"geoId/51167" +"geoId/01113" +"geoId/48143" +"geoId/51093" +"geoId/06041" +"geoId/31155" +"geoId/46035" +"geoId/42069" +"geoId/13143" +"geoId/22077" +"geoId/51141" +"geoId/39071" +"geoId/12053" +"geoId/01015" +"geoId/48057" +"geoId/26025" +"geoId/21021" +"geoId/21185" +"geoId/45063" +"geoId/08013" +"geoId/48363" +"geoId/40097" +"geoId/01125" +"geoId/04013" +"geoId/08035" +"geoId/13097" +"geoId/17041" +"geoId/20045" +"geoId/27041" +"geoId/31055" +"geoId/41019" +"geoId/55031" +"geoId/32005" +"geoId/53017" +"geoId/53057" +"geoId/22045" +"geoId/42083" +"geoId/48003" +"geoId/32001" +"geoId/49005" +"geoId/05001" +"geoId/22067" +"geoId/56041" +"geoId/45047" +"geoId/22101" +"geoId/06083" +"geoId/13055" +"geoId/01077" +"geoId/28075" +"geoId/47097" +"geoId/27109" +"geoId/48331" +"geoId/36049" +"geoId/53041" +"geoId/54041" +"geoId/12081" +"geoId/48325" +"geoId/39103" +"geoId/08093" +"geoId/30067" +"geoId/56029" +"geoId/01043" +"geoId/05139" +"geoId/17181" +"geoId/21225" +"geoId/22111" +"geoId/28145" +"geoId/37179" +"geoId/41061" +"geoId/45087" +"geoId/46127" +"geoId/39159" +"geoId/42119" +"geoId/13291" +"geoId/47173" +"geoId/53063" +"geoId/51051" +"geoId/26099" +"geoId/08043" +"geoId/56013" +"geoId/12105" +"geoId/13233" +"geoId/27119" +"geoId/29167" +"geoId/37149" +"geoId/41053" +"geoId/47139" +"geoId/55095" +"geoId/05113" +"geoId/19153" +"geoId/48373" +"geoId/48499" +"geoId/39173" +"geoId/55141" +"geoId/54107" +"geoId/37111" +"geoId/54047" +"geoId/48133" +"geoId/48005" +"geoId/24047" +"geoId/41007" +"geoId/13159" +"geoId/18073" +"geoId/19099" +"geoId/28061" +"geoId/29097" +"geoId/45053" +"geoId/48241" +"geoId/13187" +"geoId/21083" +"geoId/08101" +"geoId/26109" +"geoId/22005" +"geoId/12095" +"geoId/18117" +"geoId/36071" +"geoId/37135" +"geoId/48361" +"geoId/51137" +"geoId/50017" +"geoId/39025" +"geoId/26149" +"geoId/18157" +"geoId/12035" +"geoId/47159" +"geoId/48423" +"geoId/22095" +"geoId/04019" +"geoId/01055" +"geoId/17011" +"geoId/48257" +"geoId/29187" +"geoId/27129" +"geoId/34019" +"geoId/21043" +"geoId/40019" +"geoId/47019" +"geoId/27097" +"geoId/19133" +"geoId/12109" +"geoId/42005" +"geoId/19021" +"geoId/18179" +"geoId/12059" +"geoId/28051" +"geoId/39075" +"geoId/27013" +"geoId/47143" +"geoId/34027" +"geoId/48085" +"geoId/19139" +"geoId/29119" +"geoId/19149" +"geoId/56037" +"geoId/37165" +"geoId/05035" +"geoId/45003" +"geoId/13299" +"geoId/13257" +"geoId/40137" +"geoId/19109" +"geoId/08089" +"geoId/35035" +"geoId/02090" +"geoId/12003" +"geoId/22097" +"geoId/51005" +"geoId/37193" +"geoId/48179" +"geoId/27093" +"geoId/48007" +"geoId/19187" +"geoId/22119" +"geoId/29225" +"geoId/37161" +"geoId/47149" +"geoId/21119" +"geoId/53049" +"geoId/48117" +"geoId/51109" +"geoId/01047" +"geoId/29059" +"geoId/48113" +"geoId/19049" +"geoId/05123" +"geoId/48337" +"geoId/12127" +"geoId/27049" +"geoId/35039" +"geoId/35045" +"geoId/53055" +"geoId/32023" +"geoId/51810" +"geoId/28127" +"geoId/21213" +"geoId/47011" +"geoId/28027" +"geoId/37145" +"geoId/18091" +"geoId/13139" +"geoId/31079" +"geoId/17199" +"geoId/47187" +"geoId/48491" +"geoId/13255" +"geoId/04017" +"geoId/32007" +"geoId/42087" +"geoId/42129" +"geoId/51193" +"geoId/19043" +"geoId/27025" +"geoId/56025" +"geoId/06007" +"geoId/50027" +"geoId/39141" +"geoId/05005" +"geoId/36069" +"geoId/28115" +"geoId/40123" +"geoId/48039" +"geoId/40119" +"geoId/36097" +"geoId/37025" +"geoId/37139" +"geoId/17105" +"geoId/22063" +"geoId/36051" +"geoId/26093" +"geoId/46103" +"geoId/36089" +"geoId/28083" +"geoId/40079" +"geoId/39105" +"geoId/48355" +"geoId/19113" +"geoId/41043" +"geoId/35025" +"geoId/51047" +"geoId/21035" +"geoId/42013" +"geoId/22009" +"geoId/20209" +"geoId/53007" +"geoId/22085" +"geoId/12051" +"geoId/13245" +"geoId/37153" +"geoId/26107" +"geoId/36023" +"geoId/13131" +"geoId/40051" +"geoId/42011" +"geoId/40087" +"geoId/22047" +"geoId/01001" +"geoId/39031" +"geoId/24005" +"geoId/05091" +"geoId/29131" +"geoId/06089" +"geoId/27035" +"geoId/16039" +"geoId/01051" +"geoId/37085" +"geoId/20177" +"geoId/27085" +"geoId/51179" +"geoId/18129" +"geoId/21099" +"geoId/13147" +"geoId/12005" +"geoId/26017" +"geoId/55025" +"geoId/18039" +"geoId/45035" +"geoId/24019" +"geoId/27071" +"geoId/55013" +"geoId/18163" +"geoId/36095" +"geoId/01121" +"geoId/28049" +"geoId/13129" +"geoId/06097" +"geoId/22075" +"geoId/20035" +"geoId/19079" +"geoId/47065" +"geoId/39061" +"geoId/18057" +"geoId/47085" +"geoId/27141" +"geoId/19057" +"geoId/45031" +"geoId/39021" +"geoId/17019" +"geoId/36099" +"geoId/39147" +"geoId/01003" +"geoId/13009" +"geoId/06067" +"geoId/45019" +"geoId/01103" +"geoId/08087" +"geoId/13211" +"geoId/17137" +"geoId/18109" +"geoId/29141" +"geoId/47129" +"geoId/54065" +"geoId/05003" +"geoId/29186" +"geoId/47013" +"geoId/51031" +"geoId/56005" +"geoId/21037" +"geoId/05041" +"geoId/55101" +"geoId/48099" +"geoId/40115" +"geoId/26139" +"geoId/39123" +"geoId/37011" +"geoId/26143" +"geoId/04009" +"geoId/45071" +"geoId/39125" +"geoId/13223" +"geoId/41017" +"geoId/29183" +"geoId/26051" +"geoId/48231" +"geoId/53065" +"geoId/17075" +"geoId/06093" +"geoId/45015" +"geoId/54003" +"geoId/22089" +"geoId/35013" +"geoId/27115" +"geoId/48389" +"geoId/48467" +"geoId/26101" +"geoId/37167" +"geoId/26129" +"geoId/21025" +"geoId/29031" +"geoId/36027" +"geoId/42043" +"geoId/27057" +"geoId/21089" +"geoId/37155" +"geoId/37093" +"geoId/37001" +"geoId/20055" +"geoId/27131" +"geoId/48019" +"geoId/55137" +"geoId/41045" +"geoId/17141" +"geoId/12021" +"geoId/18167" +"geoId/48481" +"geoId/49039" +"geoId/04021" +"geoId/55113" +"geoId/22055" +"geoId/28071" +"geoId/55065" +"geoId/29107" +"geoId/55015" +"geoId/30081" +"geoId/06095" +"geoId/05017" +"geoId/06055" +"geoId/16045" +"geoId/12057" +"geoId/42109" +"geoId/16079" +"geoId/37017" +"geoId/19031" +"geoId/51550" +"geoId/27039" +"geoId/55027" +"geoId/13091" +"geoId/31053" +"geoId/21017" +"geoId/48167" +"geoId/48183" +"geoId/18165" +"geoId/35029" +"geoId/24037" +"geoId/47037" +"geoId/37057" +"geoId/29159" +"geoId/21019" +"geoId/53029" +"geoId/12017" +"geoId/22099" +"geoId/12111" +"geoId/06047" +"geoId/24035" +"geoId/12099" +"geoId/27053" +"geoId/08085" +"geoId/51075" +"geoId/51101" +"geoId/06029" +"geoId/45055" +"geoId/13127" +"geoId/34015" +"geoId/51073" +"geoId/26053" +"geoId/55061" +"geoId/48177" +"geoId/06021" diff --git a/scripts/us_usda/quickstats/golden_data/golden_summary_report.csv b/scripts/us_usda/quickstats/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..5c474e2be2 --- /dev/null +++ b/scripts/us_usda/quickstats/golden_data/golden_summary_report.csv @@ -0,0 +1,30 @@ +"NumPlaces","StatVar","ScalingFactors","Units","MeasurementMethods","MinDate","observationPeriods" +"71","Area_Farm_BarleyForGrain","[]","[Acre]","[]","2025","[P1Y]" +"2929","Count_FarmInventory_CattleAndCalves","[]","[]","[]","2024","[P1Y]" +"1715","Count_FarmInventory_BeefCows","[]","[]","[]","2024","[P1Y]" +"249","Area_Farm_CornForSilageOrGreenchop","[]","[Acre]","[]","2024","[P1Y]" +"114","Area_Farm_OtherSpringWheatForGrain","[]","[Acre]","[]","2025","[P1Y]" +"132","Area_Farm_SorghumForGrain","[]","[Acre]","[]","2024","[P1Y]" +"21","Area_Farm_DurumWheatForGrain","[]","[Acre]","[]","2025","[P1Y]" +"100","Count_FarmInventory_Layers","[]","[]","[]","2024","[P1Y]" +"21","Amount_FarmInventory_DurumWheatForGrain","[]","[Bushel]","[]","2025","[P1Y]" +"1511","Amount_FarmInventory_CornForGrain","[]","[Bushel]","[]","2024","[P1Y]" +"230","Area_Farm_OatsForGrain","[]","[Acre]","[]","2024","[P1Y]" +"727","AmountFarmInventory_WinterWheatForGrain","[]","[Bushel]","[]","2024","[P1Y]" +"114","Amount_FarmInventory_PeanutsForNuts","[]","[Pound]","[]","2024","[P1Y]" +"870","Count_FarmInventory_MilkCows","[]","[]","[]","2024","[P1Y]" +"4","Area_Farm_SweetPotatoes","[]","[Acre]","[]","2024","[P1Y]" +"114","Amount_FarmInventory_OtherSpringWheatForGrain","[]","[Bushel]","[]","2025","[P1Y]" +"7","Area_Farm_PimaCotton","[]","[Acre]","[]","2024","[P1Y]" +"249","Amount_FarmInventory_CornForSilageOrGreenchop","[]","[Ton]","[]","2024","[P1Y]" +"62","Area_Farm_Rice","[]","[Acre]","[]","2024","[P1Y]" +"62","Amount_FarmInventory_Rice","[]","[Hundredweight]","[]","2024","[P1Y]" +"316","Area_Farm_UplandCotton","[]","[Acre]","[]","2024","[P1Y]" +"71","Amount_FarmInventory_BarleyForGrain","[]","[Bushel]","[]","2025","[P1Y]" +"114","Area_Farm_PeanutsForNuts","[]","[Acre]","[]","2024","[P1Y]" +"240","Count_FarmInventory_SheepAndLambs","[]","[]","[]","2024","[P1Y]" +"100","Count_FarmInventory_HogsAndPigs","[]","[]","[]","2024","[P1Y]" +"727","Area_Farm_WinterWheatForGrain","[]","[Acre]","[]","2024","[P1Y]" +"132","Amount_FarmInventory_SorghumForGrain","[]","[Bushel]","[]","2024","[P1Y]" +"1511","Area_Farm_CornForGrain","[]","[Acre]","[]","2024","[P1Y]" +"230","Amount_FarmInventory_OatsForGrain","[]","[Bushel]","[]","2024","[P1Y]" diff --git a/scripts/us_usda/quickstats/manifest.json b/scripts/us_usda/quickstats/manifest.json index f7adcb75ba..80c9fa28c5 100644 --- a/scripts/us_usda/quickstats/manifest.json +++ b/scripts/us_usda/quickstats/manifest.json @@ -19,7 +19,8 @@ "cleaned_csv": "output/*.csv" } ], - "cron_schedule": "30 3 10 3,6,9,12 *" + "cron_schedule": "30 3 10 3,6,9,12 *", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/scripts/us_usda/quickstats/process.py b/scripts/us_usda/quickstats/process.py index 03855f7616..6adcee8c90 100644 --- a/scripts/us_usda/quickstats/process.py +++ b/scripts/us_usda/quickstats/process.py @@ -105,7 +105,7 @@ def process_survey_data(year, svs, input_dir, out_dir): logging.info(f'End, {year}, =, {end}') logging.info(f'Duration, {year}, =, {str(end - start)}') except Exception as e: - logging.fatal(f"Error while processing the data for year: {e}") + logging.error(f"Error while processing the data for year: {e}") raise RuntimeError( f"Failed to process data for year due to: {e}") from e @@ -156,7 +156,8 @@ def write_aggregate_csv(year, input_dir, out_dir): with open(f"{parts_dir}/{part_file}", 'r') as part: csv_writer.writerows(csv.DictReader(part)) except Exception as e: - logging.fatal(f"Error in write_aggregate_csv for year {year}: {e}") + logging.error(f"Error in write_aggregate_csv for year {year}: {e}") + raise def fetch_and_write(county_name, year, svs, input_dir): @@ -187,8 +188,9 @@ def fetch_and_write(county_name, year, svs, input_dir): f"No data to write for county {county_name}. Skipping file creation." ) except Exception as e: - logging.fatal( + logging.error( f"Error processing data for county: {county_name}. Error: {e}") + raise def get_survey_county_data(year, county, input_dir): @@ -234,9 +236,8 @@ def get_survey_county_data(year, county, input_dir): response = get_data(params) if response is None: logging.error( - f"get_data() returned None. Check logs for details.") - logging.warning(f"No data found for: {county}") - return {'data': []} + f"get_data() returned None for county: {county}. Raising error to prevent silent data loss.") + raise RuntimeError(f"get_data() returned None for county: {county}") with open(response_file, 'w') as f: logging.info(f"Writing response to file: {response_file}") @@ -250,10 +251,9 @@ def get_survey_county_data(year, county, input_dir): return response except Exception as e: - logging.fatal( + logging.error( f"Error while fetching data for county: {county} - {e}") - # Return a default value in case of any exception - return {'data': []} + raise MAX_REQUESTS_PER_WINDOW = 2 # Maximum requests per window (1 request) @@ -290,6 +290,21 @@ def get_data(params): try: response = requests.get(f'{API_BASE}/api_GET', params=params) + if response.status_code == 400: + try: + err_json = response.json() + err_text = str(err_json.get("error", "")).lower() + if "bad request" in err_text or "no data found" in err_text or "invalid query" in err_text: + logging.info(f"No records found for query: {params}") + return {'data': []} + except Exception: + pass + logging.warning( + f"API returned 400 status code with error: {response.text}" + ) + raise requests.exceptions.RequestException( + f"Status 400 error: {response.text}") + response.raise_for_status() if response.status_code != 200: @@ -304,6 +319,7 @@ def get_data(params): except requests.exceptions.RequestException as e: logging.error(f"Error fetching data: {e}") + raise def get_param_values(param): diff --git a/scripts/us_usda/quickstats/process_test.py b/scripts/us_usda/quickstats/process_test.py index b8339e1174..a05583745b 100644 --- a/scripts/us_usda/quickstats/process_test.py +++ b/scripts/us_usda/quickstats/process_test.py @@ -88,6 +88,55 @@ def test_write_csv(self, mock_get_api_key, mock_exists, mock_get_data): """ self.assertEqual(expected_csv_output, out.getvalue()) + @mock.patch('process.requests.get') + @mock.patch('process.is_rate_limited', return_value=False) + def test_get_data_400_no_data_found(self, mock_rate_limit, mock_requests_get): + """Tests that get_data returns {'data': []} cleanly when API returns 400 'no data found'.""" + from process import get_data + mock_resp = mock.Mock() + mock_resp.status_code = 400 + mock_resp.json.return_value = {"error": ["bad request - unsupported query / no data found"]} + mock_requests_get.return_value = mock_resp + + result = get_data({'source_desc': 'SURVEY', 'year': 2024, 'county_name': 'FAKECOUNTY'}) + self.assertEqual(result, {'data': []}) + + @mock.patch('process.requests.get') + @mock.patch('process.is_rate_limited', return_value=False) + def test_get_data_400_invalid_query(self, mock_rate_limit, mock_requests_get): + """Tests that get_data returns {'data': []} cleanly when API returns 400 'invalid query' (e.g. WASHABAUGH).""" + from process import get_data + mock_resp = mock.Mock() + mock_resp.status_code = 400 + mock_resp.json.return_value = {"error": ["bad request - invalid query"]} + mock_requests_get.return_value = mock_resp + + result = get_data({'source_desc': 'SURVEY', 'year': 2024, 'county_name': 'WASHABAUGH'}) + self.assertEqual(result, {'data': []}) + + + @mock.patch('process.requests.get') + @mock.patch('process.is_rate_limited', return_value=False) + def test_get_data_500_raises_error(self, mock_rate_limit, mock_requests_get): + """Tests that get_data raises RequestException after retries on 500 error.""" + import requests + from process import get_data + mock_resp = mock.Mock() + mock_resp.status_code = 500 + mock_resp.raise_for_status.side_effect = requests.exceptions.HTTPError("500 Server Error") + mock_requests_get.return_value = mock_resp + + with self.assertRaises(requests.exceptions.RequestException): + get_data({'source_desc': 'SURVEY', 'year': 2024, 'county_name': 'FAKECOUNTY'}) + + @mock.patch('process.get_data', return_value=None) + @mock.patch('process.get_usda_api_key', return_value='mock-api-key') + def test_get_survey_county_data_none_raises(self, mock_key, mock_get_data): + """Tests that get_survey_county_data raises RuntimeError if get_data returns None.""" + with self.assertRaises(RuntimeError): + get_survey_county_data(2025, 'FAKECOUNTY', 'testdata') + if __name__ == '__main__': absltest.main() + diff --git a/scripts/us_usda/quickstats/validation_config.json b/scripts/us_usda/quickstats/validation_config.json new file mode 100644 index 0000000000..3e4e40ae29 --- /dev/null +++ b/scripts/us_usda/quickstats/validation_config.json @@ -0,0 +1,30 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Strictly enforce historical deletion average threshold of 0.1%", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_goldens_summary_report", + "description": "Validates summary_report.csv against the golden summary data", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + }, + { + "rule_id": "check_goldens_output_csv", + "description": "Verifies the generated output CSV data matches established critical golden records", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_observations.csv", + "input_files": "../../../../output/*.csv" + } + } + ] +}