diff --git a/.gitignore b/.gitignore index d453f58..d91525d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ pgbench_log.* pg_csv--*.sql !pg_csv--*--*.sql tags +bench/data/customers-1000000.csv diff --git a/Makefile b/Makefile index 362c585..d0c37ca 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,7 @@ REGRESS_OPTS = --inputdir=test MODULE_big = $(EXTENSION) SRC = $(wildcard $(SRC_DIR)/*.c) +BENCH_DATA_DIR=bench/data ifdef BUILD_DIR OBJS = $(patsubst $(SRC_DIR)/%.c, $(BUILD_DIR)/%.o, $(SRC)) @@ -53,6 +54,8 @@ all: sql/$(EXTENSION)--$(EXTVERSION).sql $(EXTENSION).control build: $(BUILD_DIR)/$(EXTENSION).$(SHARED_EXT) sql/$(EXTENSION)--$(EXTVERSION).sql $(EXTENSION).control +bench: $(BENCH_DATA_DIR)/customers-1000000.csv + $(BUILD_DIR)/.gitignore: sql/$(EXTENSION)--$(EXTVERSION).sql $(EXTENSION).control mkdir -p $(BUILD_DIR)/extension cp $(EXTENSION).control $(BUILD_DIR)/extension @@ -62,9 +65,17 @@ $(BUILD_DIR)/.gitignore: sql/$(EXTENSION)--$(EXTVERSION).sql $(EXTENSION).contro $(BUILD_DIR)/%.o: $(SRC_DIR)/%.c $(BUILD_DIR)/.gitignore $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ +$(BUILD_DIR)/pg_csv.o: $(SRC_DIR)/csv.h $(SRC_DIR)/cparsec.h +src/pg_csv.o: $(SRC_DIR)/csv.h $(SRC_DIR)/cparsec.h + $(BUILD_DIR)/$(EXTENSION).$(SHARED_EXT): $(EXTENSION).$(SHARED_EXT) mv $? $@ +$(BENCH_DATA_DIR)/customers-1000000.csv: $(BENCH_DATA_DIR)/customers-1000000.7z + 7z e -y $< -o$(BENCH_DATA_DIR) $(notdir $@) +# needed to not decompress everytime, 7z preserves archive original timestamp and that trips up make + touch $@ + sql/$(EXTENSION)--$(EXTVERSION).sql: sql/$(EXTENSION).sql cp $< $@ diff --git a/README.md b/README.md index 3c73b3c..d67ac91 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Postgres has CSV support on the [COPY](https://www.postgresql.org/docs/current/s `pg_csv` offers flexible CSV processing as a solution. -- Includes a CSV aggregate that composes with SQL expressions. +- Includes CSV export/import functions that compose with SQL expressions. - Native C extension, x2 times faster than SQL queries that try to output CSV - No dependencies except Postgres. @@ -29,7 +29,9 @@ To install the extension: create extension pg_csv; ``` -## csv_agg +## CSV Export + +### csv_agg Aggregate that builds a CSV respecting [RFC 4180](https://www.ietf.org/rfc/rfc4180.txt), quoting as required. @@ -146,6 +148,46 @@ FROM projects x; (1 row) ``` +## CSV Import + +### csv_read + +The `csv_read` function can read inline text values, respecting quoting and escaping inside quotes. + +```sql +select id, name +from csv_read(null::projects, E'1,IOS,4\n2,"Win""dows",4') where id = 2; + id | name +----+---------- + 2 | Win"dows +``` + +It can also be used to read from files inside the data directory. + +```sql +select "First Name", "Company", "Website" +from csv_read( + null::customers, + pg_read_file('data/customers-100.csv') +) +where "Index" = '4'; + Index | Customer Id | First Name | Last Name | Company | City | Country | Phone 1 | Phone 2 | Email | Subscription Date | Website +-------+---------------------------------+----------------------------+-----------+---------+------+---------+---------+---------+-------+-------------------+--------- + Linda | Dominguez, Mcmillan and Donovan | http://www.good-lyons.com/ | | | | | | | | | +``` + +This combines as usual with the `insert` statement. + +```sql +insert into customers +select * +from csv_read( + null::customers, + pg_read_file('data/customers-100.csv') +) +limit 10; -- you can use LIMIT, OFFSET or WHERE +``` + ## Limitations - For large bulk exports and imports, `COPY ... CSV` should still be preferred as it's more memory efficient due to streaming support. diff --git a/bench/csv_read.sql b/bench/csv_read.sql new file mode 100644 index 0000000..8d2da0d --- /dev/null +++ b/bench/csv_read.sql @@ -0,0 +1,8 @@ +truncate customers_csv; + +insert into customers_csv +select * +from csv_read( + null::customers_csv, + pg_read_file('data/customers-1000000.csv') +); diff --git a/bench/data/README.md b/bench/data/README.md new file mode 100644 index 0000000..fcde73c --- /dev/null +++ b/bench/data/README.md @@ -0,0 +1,7 @@ +The CSV samples were obtained from https://github.com/datablist/sample-csv-files. + +To reduce its size, the `customers-1000000.csv` example was compressed with: + +``` +7z a -t7z -m0=PPMd -mx=9 customers-1000000.7z customers-1000000.csv +``` diff --git a/bench/data/customers-100.csv b/bench/data/customers-100.csv new file mode 100644 index 0000000..468ba6b --- /dev/null +++ b/bench/data/customers-100.csv @@ -0,0 +1,100 @@ +1,DD37Cf93aecA6Dc,Sheryl,Baxter,Rasmussen Group,East Leonard,Chile,229.077.5154,397.884.0519x718,zunigavanessa@smith.info,2020-08-24,http://www.stephenson.com/ +2,1Ef7b82A4CAAD10,Preston,Lozano,Vega-Gentry,East Jimmychester,Djibouti,5153435776,686-620-1820x944,vmata@colon.com,2021-04-23,http://www.hobbs.com/ +3,6F94879bDAfE5a6,Roy,Berry,Murillo-Perry,Isabelborough,Antigua and Barbuda,+1-539-402-0259,(496)978-3969x58947,beckycarr@hogan.com,2020-03-25,http://www.lawrence.com/ +4,5Cef8BFA16c5e3c,Linda,Olsen,"Dominguez, Mcmillan and Donovan",Bensonview,Dominican Republic,001-808-617-6467x12895,+1-813-324-8756,stanleyblackwell@benson.org,2020-06-02,http://www.good-lyons.com/ +5,053d585Ab6b3159,Joanna,Bender,"Martin, Lang and Andrade",West Priscilla,Slovakia (Slovak Republic),001-234-203-0635x76146,001-199-446-3860x3486,colinalvarado@miles.net,2021-04-17,https://goodwin-ingram.com/ +6,2d08FB17EE273F4,Aimee,Downs,Steele Group,Chavezborough,Bosnia and Herzegovina,(283)437-3886x88321,999-728-1637,louis27@gilbert.com,2020-02-25,http://www.berger.net/ +7,EA4d384DfDbBf77,Darren,Peck,"Lester, Woodard and Mitchell",Lake Ana,Pitcairn Islands,(496)452-6181x3291,+1-247-266-0963x4995,tgates@cantrell.com,2021-08-24,https://www.le.com/ +8,0e04AFde9f225dE,Brett,Mullen,"Sanford, Davenport and Giles",Kimport,Bulgaria,001-583-352-7197x297,001-333-145-0369,asnow@colon.com,2021-04-12,https://hammond-ramsey.com/ +9,C2dE4dEEc489ae0,Sheryl,Meyers,Browning-Simon,Robersonstad,Cyprus,854-138-4911x5772,+1-448-910-2276x729,mariokhan@ryan-pope.org,2020-01-13,https://www.bullock.net/ +10,8C2811a503C7c5a,Michelle,Gallagher,Beck-Hendrix,Elaineberg,Timor-Leste,739.218.2516x459,001-054-401-0347x617,mdyer@escobar.net,2021-11-08,https://arias.com/ +11,216E205d6eBb815,Carl,Schroeder,"Oconnell, Meza and Everett",Shannonville,Guernsey,637-854-0256x825,114.336.0784x788,kirksalas@webb.com,2021-10-20,https://simmons-hurley.com/ +12,CEDec94deE6d69B,Jenna,Dodson,"Hoffman, Reed and Mcclain",East Andrea,Vietnam,(041)737-3846,+1-556-888-3485x42608,mark42@robbins.com,2020-11-29,http://www.douglas.net/ +13,e35426EbDEceaFF,Tracey,Mata,Graham-Francis,South Joannamouth,Togo,001-949-844-8787,(855)713-8773,alex56@walls.org,2021-12-02,http://www.beck.com/ +14,A08A8aF8BE9FaD4,Kristine,Cox,Carpenter-Cook,Jodyberg,Sri Lanka,786-284-3358x62152,+1-315-627-1796x8074,holdenmiranda@clarke.com,2021-02-08,https://www.brandt.com/ +15,6fEaA1b7cab7B6C,Faith,Lutz,Carter-Hancock,Burchbury,Singapore,(781)861-7180x8306,207-185-3665,cassieparrish@blevins-chapman.net,2022-01-26,http://stevenson.org/ +16,8cad0b4CBceaeec,Miranda,Beasley,Singleton and Sons,Desireeshire,Oman,540.085.3135x185,+1-600-462-6432x21881,vduncan@parks-hardy.com,2022-04-12,http://acosta.org/ +17,a5DC21AE3a21eaA,Caroline,Foley,Winters-Mendoza,West Adriennestad,Western Sahara,936.222.4746x9924,001-469-948-6341x359,holtgwendolyn@watson-davenport.com,2021-03-10,http://www.benson-roth.com/ +18,F8Aa9d6DfcBeeF8,Greg,Mata,Valentine LLC,Lake Leslie,Mozambique,(701)087-2415,(195)156-1861x26241,jaredjuarez@carroll.org,2022-03-26,http://pitts-cherry.com/ +19,F160f5Db3EfE973,Clifford,Jacobson,Simon LLC,Harmonview,South Georgia and the South Sandwich Islands,001-151-330-3524x0469,(748)477-7174,joseph26@jacobson.com,2020-09-24,https://mcconnell.com/ +20,0F60FF3DdCd7aB0,Joanna,Kirk,Mays-Mccormick,Jamesshire,French Polynesia,(266)131-7001x711,(283)312-5579x11543,tuckerangie@salazar.net,2021-09-24,https://www.camacho.net/ +21,9F9AdB7B8A6f7F2,Maxwell,Frye,Patterson Inc,East Carly,Malta,423.262.3059,202-880-0688x7491,fgibson@drake-webb.com,2022-01-12,http://www.roberts.com/ +22,FBd0Ded4F02a742,Kiara,Houston,"Manning, Hester and Arroyo",South Alvin,Netherlands,001-274-040-3582x10611,+1-528-175-0973x4684,blanchardbob@wallace-shannon.com,2020-09-15,https://www.reid-potts.com/ +23,2FB0FAA1d429421,Colleen,Howard,Greer and Sons,Brittanyview,Paraguay,1935085151,(947)115-7711x5488,rsingleton@ryan-cherry.com,2020-08-19,http://paul.biz/ +24,010468dAA11382c,Janet,Valenzuela,Watts-Donaldson,Veronicamouth,Lao People's Democratic Republic,354.259.5062x7538,500.433.2022,stefanie71@spence.com,2020-09-08,https://moreno.biz/ +25,eC1927Ca84E033e,Shane,Wilcox,Tucker LLC,Bryanville,Albania,(429)005-9030x11004,541-116-4501,mariah88@santos.com,2021-04-06,https://www.ramos.com/ +26,09D7D7C8Fe09aea,Marcus,Moody,Giles Ltd,Kaitlyntown,Panama,674-677-8623,909-277-5485x566,donnamullins@norris-barrett.org,2022-05-24,https://www.curry.com/ +27,aBdfcF2c50b0bfD,Dakota,Poole,Simmons Group,Michealshire,Belarus,(371)987-8576x4720,071-152-1376,stacey67@fields.org,2022-02-20,https://sanford-wilcox.biz/ +28,b92EBfdF8a3f0E6,Frederick,Harper,"Hinton, Chaney and Stokes",South Marissatown,Switzerland,+1-077-121-1558x0687,264.742.7149,jacobkhan@bright.biz,2022-05-26,https://callahan.org/ +29,3B5dAAFA41AFa22,Stefanie,Fitzpatrick,Santana-Duran,Acevedoville,Saint Vincent and the Grenadines,(752)776-3286,+1-472-021-4814x85074,wterrell@clark.com,2020-07-30,https://meyers.com/ +30,EDA69ca7a6e96a2,Kent,Bradshaw,Sawyer PLC,North Harold,Tanzania,+1-472-143-5037x884,126.922.6153,qjimenez@boyd.com,2020-04-26,http://maynard-ho.com/ +31,64DCcDFaB9DFd4e,Jack,Tate,"Acosta, Petersen and Morrow",West Samuel,Zimbabwe,965-108-4406x20714,046.906.1442x6784,gfigueroa@boone-zavala.com,2021-09-15,http://www.hawkins-ramsey.com/ +32,679c6c83DD872d6,Tom,Trujillo,Mcgee Group,Cunninghamborough,Denmark,416-338-3758,(775)890-7209,tapiagreg@beard.info,2022-01-13,http://www.daniels-klein.com/ +33,7Ce381e4Afa4ba9,Gabriel,Mejia,Adkins-Salinas,Port Annatown,Liechtenstein,4077245425,646.044.0696x66800,coleolson@jennings.net,2021-04-24,https://patel-hanson.info/ +34,A09AEc6E3bF70eE,Kaitlyn,Santana,Herrera Group,New Kaitlyn,United States of America,6303643286,447-710-6202x07313,georgeross@miles.org,2021-09-21,http://pham.com/ +35,aA9BAFfBc3710fe,Faith,Moon,"Waters, Chase and Aguilar",West Marthaburgh,Bahamas,+1-586-217-0359x6317,+1-818-199-1403,willistonya@randolph-baker.com,2021-11-03,https://spencer-charles.info/ +36,E11dfb2DB8C9f72,Tammie,Haley,"Palmer, Barnes and Houston",East Teresa,Belize,001-276-734-4113x6087,(430)300-8770,harrisisaiah@jenkins.com,2022-01-04,http://evans-simon.com/ +37,889eCf90f68c5Da,Nicholas,Sosa,Jordan Ltd,South Hunter,Uruguay,(661)425-6042,975-998-1519,fwolfe@dorsey.com,2021-08-10,https://www.fleming-richards.com/ +38,7a1Ee69F4fF4B4D,Jordan,Gay,Glover and Sons,South Walter,Solomon Islands,7208417020,8035336772,tiffanydavies@harris-mcfarland.org,2021-02-24,http://www.lee.org/ +39,dca4f1D0A0fc5c9,Bruce,Esparza,Huerta-Mclean,Poolefurt,Montenegro,559-529-4424,001-625-000-7132x0367,preese@frye-vega.com,2021-10-22,http://www.farley.org/ +40,17aD8e2dB3df03D,Sherry,Garza,Anderson Ltd,West John,Poland,001-067-713-6440x158,(978)289-8785x5766,ann48@miller.com,2021-11-01,http://spence.com/ +41,2f79Cd309624Abb,Natalie,Gentry,Monroe PLC,West Darius,Dominican Republic,830.996.8238,499.122.5415,tcummings@fitzpatrick-ashley.com,2020-10-10,http://www.dorsey.biz/ +42,6e5ad5a5e2bB5Ca,Bryan,Dunn,Kaufman and Sons,North Jimstad,Burkina Faso,001-710-802-5565,078.699.8982x13881,woodwardandres@phelps.com,2021-09-08,http://www.butler.com/ +43,7E441b6B228DBcA,Wayne,Simpson,Perkins-Trevino,East Rebekahborough,Bolivia,(344)156-8632x1869,463-445-3702x38463,barbarapittman@holder.com,2020-12-13,https://gillespie-holder.com/ +44,D3fC11A9C235Dc6,Luis,Greer,Cross PLC,North Drew,Bulgaria,001-336-025-6849x701,684.698.2911x6092,bstuart@williamson-mcclure.com,2022-05-15,https://fletcher-nielsen.com/ +45,30Dfa48fe5Ede78,Rhonda,Frost,"Herrera, Shepherd and Underwood",Lake Lindaburgh,Monaco,(127)081-9339,+1-431-028-3337x3492,zkrueger@wolf-chavez.net,2021-12-06,http://www.khan.com/ +46,fD780ED8dbEae7B,Joanne,Montes,"Price, Sexton and Mcdaniel",Gwendolynview,Palau,(897)726-7952,(467)886-9467x5721,juan80@henson.net,2020-07-01,http://ochoa.com/ +47,300A40d3ce24bBA,Geoffrey,Guzman,Short-Wiggins,Zimmermanland,Uzbekistan,975.235.8921x269,(983)188-6873,bauercrystal@gay.com,2020-04-23,https://decker-kline.com/ +48,283DFCD0Dba40aF,Gloria,Mccall,"Brennan, Acosta and Ramos",North Kerriton,Ghana,445-603-6729,001-395-959-4736x4524,bartlettjenna@zuniga-moss.biz,2022-03-11,http://burgess-frank.com/ +49,F4Fc91fEAEad286,Brady,Cohen,Osborne-Erickson,North Eileenville,United Arab Emirates,741.849.0139x524,+1-028-691-7497x0894,mccalltyrone@durham-rose.biz,2022-03-10,http://hammond-barron.com/ +50,80F33Fd2AcebF05,Latoya,Mccann,"Hobbs, Garrett and Sanford",Port Sergiofort,Belarus,(530)287-4548x29481,162-234-0249x32790,bobhammond@barry.biz,2021-12-02,https://www.burton.com/ +51,Aa20BDe68eAb0e9,Gerald,Hawkins,"Phelps, Forbes and Koch",New Alberttown,Canada,+1-323-239-1456x96168,(092)508-0269,uwarner@steele-arias.com,2021-03-19,https://valenzuela.com/ +52,e898eEB1B9FE22b,Samuel,Crawford,"May, Goodwin and Martin",South Jasmine,Algeria,802-242-7457,626.116.9535x8578,xpittman@ritter-carney.net,2021-03-27,https://guerrero.org/ +53,faCEF517ae7D8eB,Patricia,Goodwin,"Christian, Winters and Ellis",Cowanfort,Swaziland,322.549.7139x70040,(111)741-4173,vaughanchristy@lara.biz,2021-03-08,http://clark.info/ +54,c09952De6Cda8aA,Stacie,Richard,Byrd Inc,New Deborah,Madagascar,001-622-948-3641x24810,001-731-168-2893x8891,clinton85@colon-arias.org,2020-10-15,https://kim.com/ +55,f3BEf3Be028166f,Robin,West,"Nixon, Blackwell and Sosa",Wallstown,Ecuador,698.303.4267,001-683-837-7651x525,greenemiranda@zimmerman.com,2022-01-13,https://www.mora.com/ +56,C6F2Fc6a7948a4e,Ralph,Haas,Montes PLC,Lake Ellenchester,Palestinian Territory,2239271999,001-962-434-0867x649,goodmancesar@figueroa.biz,2020-05-25,http://may.com/ +57,c8FE57cBBdCDcb2,Phyllis,Maldonado,Costa PLC,Lake Whitney,Saint Barthelemy,4500370767,001-508-064-6725x017,yhanson@warner-diaz.org,2021-01-25,http://www.bernard.com/ +58,B5acdFC982124F2,Danny,Parrish,Novak LLC,East Jaredbury,United Arab Emirates,(669)384-8597x8794,506.731.5952x571,howelldarren@house-cohen.com,2021-03-17,http://www.parsons-hudson.com/ +59,8c7DdF10798bCC3,Kathy,Hill,"Moore, Mccoy and Glass",Selenabury,South Georgia and the South Sandwich Islands,001-171-716-2175x310,888.625.0654,ncamacho@boone-simmons.org,2020-11-15,http://hayden.com/ +60,C681dDd0cc422f7,Kelli,Hardy,Petty Ltd,Huangfort,Sao Tome and Principe,020.324.2191x2022,424-157-8216,kristopher62@oliver.com,2020-12-20,http://www.kidd.com/ +61,a940cE42e035F28,Lynn,Pham,"Brennan, Camacho and Tapia",East Pennyshire,Portugal,846.468.6834x611,001-248-691-0006,mpham@rios-guzman.com,2020-08-21,https://www.murphy.com/ +62,9Cf5E6AFE0aeBfd,Shelley,Harris,"Prince, Malone and Pugh",Port Jasminborough,Togo,423.098.0315x8373,+1-386-458-8944x15194,zachary96@mitchell-bryant.org,2020-12-10,https://www.ryan.com/ +63,aEcbe5365BbC67D,Eddie,Jimenez,Caldwell Group,West Kristine,Ethiopia,+1-235-657-1073x6306,(026)401-7353x2417,kristiwhitney@bernard.com,2022-03-24,http://cherry.com/ +64,FCBdfCEAe20A8Dc,Chloe,Hutchinson,Simon LLC,South Julia,Netherlands,981-544-9452,+1-288-552-4666x060,leah85@sutton-terrell.com,2022-05-15,https://mitchell.info/ +65,636cBF0835E10ff,Eileen,Lynch,"Knight, Abbott and Hubbard",Helenborough,Liberia,+1-158-951-4131x53578,001-673-779-6713x680,levigiles@vincent.com,2021-01-02,http://mckay.com/ +66,fF1b6c9E8Fbf1ff,Fernando,Lambert,Church-Banks,Lake Nancy,Lithuania,497.829.9038,3863743398,fisherlinda@schaefer.net,2021-04-23,https://www.vang.com/ +67,2A13F74EAa7DA6c,Makayla,Cannon,Henderson Inc,Georgeport,New Caledonia,001-215-801-6392x46009,027-609-6460,scottcurtis@hurley.biz,2020-01-20,http://www.velazquez.net/ +68,a014Ec1b9FccC1E,Tom,Alvarado,Donaldson-Dougherty,South Sophiaberg,Kiribati,(585)606-2980x2258,730-797-3594x5614,nicholsonnina@montgomery.info,2020-08-18,http://odom-massey.com/ +69,421a109cABDf5fa,Virginia,Dudley,Warren Ltd,Hartbury,French Southern Territories,027.846.3705x14184,+1-439-171-1846x4636,zvalencia@phelps.com,2021-01-31,http://hunter-esparza.com/ +70,CC68FD1D3Bbbf22,Riley,Good,Wade PLC,Erikaville,Canada,6977745822,855-436-7641,alex06@galloway.com,2020-02-03,http://conway.org/ +71,CBCd2Ac8E3eBDF9,Alexandria,Buck,Keller-Coffey,Nicolasfort,Iran,078-900-4760x76668,414-112-8700x68751,lee48@manning.com,2021-02-20,https://ramsey.org/ +72,Ef859092FbEcC07,Richard,Roth,Conway-Mcbride,New Jasmineshire,Morocco,581-440-6539,9857827463,aharper@maddox-townsend.org,2020-02-23,https://www.brooks.com/ +73,F560f2d3cDFb618,Candice,Keller,Huynh and Sons,East Summerstad,Zimbabwe,001-927-965-8550x92406,001-243-038-4271x53076,buckleycory@odonnell.net,2020-08-22,https://www.lucero.com/ +74,A3F76Be153Df4a3,Anita,Benson,Parrish Ltd,Skinnerport,Russian Federation,874.617.5668x69878,(399)820-6418x0071,angie04@oconnell.com,2020-02-09,http://oconnor.com/ +75,D01Af0AF7cBbFeA,Regina,Stein,Guzman-Brown,Raystad,Solomon Islands,001-469-848-0724x4407,001-085-360-4426x00357,zrosario@rojas-hardin.net,2022-01-15,http://www.johnston.info/ +76,d40e89dCade7b2F,Debra,Riddle,"Chang, Aguirre and Leblanc",Colinhaven,United States Virgin Islands,+1-768-182-6014x14336,(303)961-4491,shieldskerry@robles.com,2020-07-11,http://kaiser.info/ +77,BF6a1f9bd1bf8DE,Brittany,Zuniga,Mason-Hester,West Reginald,Kyrgyz Republic,(050)136-9025,001-480-851-2496x0157,mchandler@cochran-huerta.org,2021-07-24,http://www.boyle.com/ +78,FfaeFFbbbf280db,Cassidy,Mcmahon,"Mcguire, Huynh and Hopkins",Lake Sherryborough,Myanmar,5040771311,684-682-0021x1326,katrinalane@fitzgerald.com,2020-10-21,https://hurst.com/ +79,CbAE1d1e9a8dCb1,Laurie,Pennington,"Sanchez, Marsh and Hale",Port Katherineville,Dominica,007.155.3406x553,+1-809-862-5566x277,cookejill@powell.com,2020-06-08,http://www.hebert.com/ +80,A7F85c1DE4dB87f,Alejandro,Blair,"Combs, Waller and Durham",Thomasland,Iceland,(690)068-4641x51468,555.509.8691x2329,elizabethbarr@ewing.com,2020-09-19,https://mercado-blevins.com/ +81,D6CEAfb3BDbaa1A,Leslie,Jennings,Blankenship-Arias,Coreybury,Micronesia,629.198.6346,075.256.0829,corey75@wiggins.com,2021-11-13,https://www.juarez.com/ +82,Ebdb6F6F7c90b69,Kathleen,Mckay,"Coffey, Lamb and Johnson",Lake Janiceton,Saint Vincent and the Grenadines,(733)910-9968,(691)247-4128x0665,chloelester@higgins-wilkinson.com,2021-09-12,http://www.owens-mooney.com/ +83,E8E7e8Cfe516ef0,Hunter,Moreno,Fitzpatrick-Lawrence,East Clinton,Isle of Man,(733)833-6754,001-761-013-7121,isaac26@benton-finley.com,2020-12-28,http://walls.info/ +84,78C06E9b6B3DF20,Chad,Davidson,Garcia-Jimenez,South Joshuashire,Oman,8275702958,(804)842-4715,justinwalters@jimenez.com,2021-11-15,http://www.garner-oliver.com/ +85,03A1E62ADdeb31c,Corey,Holt,"Mcdonald, Bird and Ramirez",New Glenda,Fiji,001-439-242-4986x7918,3162708934,maurice46@morgan.com,2020-02-18,http://www.watson.com/ +86,C6763c99d0bd16D,Emma,Cunningham,Stephens Inc,North Jillianview,New Zealand,128-059-0206x60217,(312)164-4545x2284,walter83@juarez.org,2022-05-13,http://www.reid.info/ +87,ebe77E5Bf9476CE,Duane,Woods,Montoya-Miller,Lyonsberg,Maldives,(636)544-7783x7288,(203)287-1003x5932,kmercer@wagner.com,2020-07-21,http://murray.org/ +88,E4Bbcd8AD81fC5f,Alison,Vargas,"Vaughn, Watts and Leach",East Cristinabury,Benin,365-273-8144,053-308-7653x6287,vcantu@norton.com,2020-11-10,http://mason.info/ +89,efeb73245CDf1fF,Vernon,Kane,Carter-Strickland,Thomasfurt,Yemen,114-854-1159x555,499-608-4612,hilljesse@barrett.info,2021-04-15,http://www.duffy-hensley.net/ +90,37Ec4B395641c1E,Lori,Flowers,Decker-Mcknight,North Joeburgh,Namibia,679.415.1210,945-842-3659x4581,tyrone77@valenzuela.info,2021-01-09,http://www.deleon-crosby.com/ +91,5ef6d3eefdD43bE,Nina,Chavez,Byrd-Campbell,Cassidychester,Bhutan,053-344-3205,+1-330-920-5422x571,elliserica@frank.com,2020-03-26,https://www.pugh.com/ +92,98b3aeDcC3B9FF3,Shane,Foley,Rocha-Hart,South Dannymouth,Hungary,+1-822-569-0302,001-626-114-5844x55073,nsteele@sparks.com,2021-07-06,https://www.holt-sparks.com/ +93,aAb6AFc7AfD0fF3,Collin,Ayers,Lamb-Peterson,South Lonnie,Anguilla,404-645-5351x012,001-257-582-8850x8516,dudleyemily@gonzales.biz,2021-06-29,http://www.ruiz.com/ +94,54B5B5Fe9F1B6C5,Sherry,Young,"Lee, Lucero and Johnson",Frankchester,Solomon Islands,158-687-1764,(438)375-6207x003,alan79@gates-mclaughlin.com,2021-04-04,https://travis.net/ +95,BE91A0bdcA49Bbc,Darrell,Douglas,"Newton, Petersen and Mathis",Daisyborough,Mali,001-084-845-9524x1777,001-769-564-6303,grayjean@lowery-good.com,2022-02-17,https://banks.biz/ +96,cb8E23e48d22Eae,Karl,Greer,Carey LLC,East Richard,Guyana,(188)169-1674x58692,001-841-293-3519x614,hhart@jensen.com,2022-01-30,http://hayes-perez.com/ +97,CeD220bdAaCfaDf,Lynn,Atkinson,"Ware, Burns and Oneal",New Bradview,Sri Lanka,+1-846-706-2218,605.413.3198,vkemp@ferrell.com,2021-07-10,https://novak-allison.com/ +98,28CDbC0dFe4b1Db,Fred,Guerra,Schmitt-Jones,Ortegaland,Solomon Islands,+1-753-067-8419x7170,+1-632-666-7507x92121,swagner@kane.org,2021-09-18,https://www.ross.com/ +99,c23d1D9EE8DEB0A,Yvonne,Farmer,Fitzgerald-Harrell,Lake Elijahview,Aruba,(530)311-9786,001-869-452-0943x12424,mccarthystephen@horn-green.biz,2021-08-11,http://watkins.info/ +100,2354a0E336A91A1,Clarence,Haynes,"Le, Nash and Cross",Judymouth,Honduras,(753)813-6941,783.639.1472,colleen91@faulkner.biz,2020-03-11,http://www.hatfield-saunders.net/ diff --git a/bench/data/customers-1000000.7z b/bench/data/customers-1000000.7z new file mode 100644 index 0000000..4758023 Binary files /dev/null and b/bench/data/customers-1000000.7z differ diff --git a/bench/init.sql b/bench/init.sql index 84cab30..95428b5 100644 --- a/bench/init.sql +++ b/bench/init.sql @@ -3,6 +3,21 @@ create extension if not exists pg_csv; +create unlogged table customers_csv ( + "Index" text, + "Customer Id" text, + "First Name" text, + "Last Name" text, + "Company" text, + "City" text, + "Country" text, + "Phone 1" text, + "Phone 2" text, + "Email" text, + "Subscription Date" text, + "Website" text +); + CREATE TABLE customers ( customer_id CHAR(5) PRIMARY KEY, company_name TEXT NOT NULL, diff --git a/bench/native_copy.sql b/bench/native_copy.sql new file mode 100644 index 0000000..87da2f1 --- /dev/null +++ b/bench/native_copy.sql @@ -0,0 +1,6 @@ +truncate customers_csv; + +copy customers_csv from 'data/customers-1000000.csv' with ( + format csv, + header true +); diff --git a/shell.nix b/shell.nix index de5b26b..319d876 100644 --- a/shell.nix +++ b/shell.nix @@ -26,12 +26,20 @@ mkShellNoCC { pg_ver=$1 + make bench > /dev/null + for file in ./bench/*.sql; do if [ "$file" = ./bench/init.sql ]; then continue fi + duration=30 + if [ "$file" = ./bench/native_copy.sql ] || + [ "$file" = ./bench/csv_read.sql ]; then + duration=120 + fi + cat <kind != CPC_SLICE) return cpc_res_err(rest, "csv: expected slice", NULL); + + CpcSlice s = v->as.slice; + // Leave non-quoted slices unchanged, only unescape fully quoted + if (s.len < 2 || s.ptr[0] != '"' || s.ptr[s.len - 1] != '"') + return cpc_res_ok(cpc_val_slice(s), rest); + + char *out = (char *)s.ptr + 1; + size_t dst = 0; + size_t src = 1; + size_t end = s.len - 1; + while (src < end) { + char c = s.ptr[src]; + // Rewrite the quoted field in place, collapsing doubled quotes to one char + if (c == '"' && (src + 1) < end && s.ptr[src + 1] == '"') { + out[dst++] = '"'; + src += 2; + } else { + out[dst++] = c; + src++; + } + } + + return cpc_res_ok(cpc_val_slice((CpcSlice){.ptr = out, .len = dst}), rest); +} + +extern CpcResult csvRow(CpcSlice input, CpcArena *A, const char *err); + +// semicolons are added just for to not make clang-format crazy +static inline CPC_TAKE_QUOTED(quoted, '"', '"'); +static inline CPC_MAP(quotedField, quoted, unescape_quoted); +static inline CPC_TAKE_TILL_ONE_OF(unquotedField, ",\r\n"); +static inline CPC_STRING(comma, ","); +static inline CPC_ALT(field_, quotedField, unquotedField); +static inline CPC_LABEL(field, field_, "field"); +static inline CPC_SEP_BY_1(record, field, comma); +static inline CPC_ALT(lineEnd_, CPC_END_OF_LINE_, CPC_EOF_); +static inline CPC_LABEL(lineEnd, lineEnd_, "expected newline or end of input"); +CPC_LEFT(csvRow, record, lineEnd); + +#endif diff --git a/src/pg_csv.c b/src/pg_csv.c index 4937a86..6b0d3c7 100644 --- a/src/pg_csv.c +++ b/src/pg_csv.c @@ -4,9 +4,63 @@ #include "pg_prelude.h" #include "aggs.h" +#include "csv.h" +#include PG_MODULE_MAGIC; +static int count_visible_columns(TupleDesc tupdesc) { + int visible = 0; + + for (int i = 0; i < tupdesc->natts; i++) { + if (!TupleDescAttr(tupdesc, i)->attisdropped) visible++; + } + + return visible; +} + +// We need a cstring to pass to functions like InputFunctionCall, as unfortunately it doesn't allow +// passing a length. To avoid copying into a new buffer and appending it a `\0`, we take advantage +// of some facts: +// - The slice always point into a mutable input buffer +// - The byte immediately after the slice is still within that same buffer. either a csv +// delimiter/newline or the trailing '\0' from text_to_cstring(). +static char *csv_slice_to_cstr(CpcSlice field) { + char *cstr = (char *)field.ptr; + cstr[field.len] = '\0'; + return cstr; +} + +// Convert the parsed field slices into typed Datums for one output tuple +static void csv_row_to_datums(MemoryContext row_ctx, const CpcArena *arena, const CpcValue *row, + size_t field_count, int natts, const int *visible_attnos, + FmgrInfo *infuncs, Oid *ioparams, int32 *typmods, Datum *values, + bool *nulls) { + MemoryContextReset(row_ctx); + MemoryContext oldctx = MemoryContextSwitchTo(row_ctx); + + for (int i = 0; i < natts; i++) { + values[i] = (Datum)0; + nulls[i] = true; + } + + for (int col = 0; col < (int)field_count; col++) { + const CpcValue *field = cpc_val_list_at(arena, row, col); + int att_index = visible_attnos[col]; + + if (field == NULL || !cpc_is_slice(field)) + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("missing row column data"))); + + if (field->as.slice.len == 0) continue; + + values[att_index] = InputFunctionCall(&infuncs[att_index], csv_slice_to_cstr(field->as.slice), + ioparams[att_index], typmods[att_index]); + nulls[att_index] = false; + } + + MemoryContextSwitchTo(oldctx); +} + // aggregate final function PG_FUNCTION_INFO_V1(csv_agg_finalfn); Datum csv_agg_finalfn(PG_FUNCTION_ARGS) { @@ -128,3 +182,106 @@ Datum csv_agg_transfn(PG_FUNCTION_ARGS) { PG_RETURN_POINTER(state); } + +PG_FUNCTION_INFO_V1(csv_read); +Datum csv_read(PG_FUNCTION_ARGS) { + if (PG_ARGISNULL(1)) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("input must not be NULL"))); + + ReturnSetInfo *rsinfo = (ReturnSetInfo *)fcinfo->resultinfo; + + Oid base_type = get_fn_expr_argtype(fcinfo->flinfo, 0); + + TupleDesc srcdesc = lookup_rowtype_tupdesc(base_type, -1); + TupleDesc outdesc = BlessTupleDesc(CreateTupleDescCopy(srcdesc)); + int natts = srcdesc->natts; + int vnatts = count_visible_columns(srcdesc); + + if (vnatts == 0) + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("row type has no columns"))); + + FmgrInfo *infuncs = palloc0(mul_size(natts, sizeof(FmgrInfo))); + Oid *ioparams = palloc0(mul_size(natts, sizeof(Oid))); + int32 *typmods = palloc0(mul_size(natts, sizeof(int32))); + + for (int i = 0; i < natts; i++) { + Form_pg_attribute att = TupleDescAttr(srcdesc, i); + Oid inoid; + + if (att->attisdropped) continue; + + getTypeInputInfo(att->atttypid, &inoid, &ioparams[i]); + fmgr_info(inoid, &infuncs[i]); + typmods[i] = att->atttypmod; + } + + int *visible_attnos = palloc(mul_size(vnatts, sizeof(int))); + int visible_idx = 0; + + for (int i = 0; i < natts; i++) { + if (!TupleDescAttr(srcdesc, i)->attisdropped) visible_attnos[visible_idx++] = i; + } + + MemoryContext per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + MemoryContext oldctx = MemoryContextSwitchTo(per_query_ctx); + + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setDesc = outdesc; + rsinfo->setResult = tuplestore_begin_heap(true, false, work_mem); + + // Parse the input text row by row, then materialize each row into the tuplestore. + CpcSlice remaining = cpc_slice_from_cstr(text_to_cstring(PG_GETARG_TEXT_PP(1))); + + // MaxHeapAttributeNumber is the maximum number of columns a table can have + CpcValue arena_items[MaxHeapAttributeNumber]; + CpcArena arena = {0}; + Datum *values = palloc(mul_size(natts, sizeof(Datum))); + bool *nulls = palloc(mul_size(natts, sizeof(bool))); + MemoryContext row_ctx = + AllocSetContextCreate(per_query_ctx, "csv_read row", ALLOCSET_DEFAULT_SIZES); + int64 row_number = 0; + + cpc_arena_init(&arena, arena_items, MaxHeapAttributeNumber, NULL); + + while (remaining.len > 0) { + // Reuse the arena for each csv row + cpc_arena_reset(&arena); + + CpcResult parsed = CPC_PARSE(csvRow, remaining, &arena); + + if (!parsed.ok) { + if (parsed.err.kind == CPC_ERR_ARENA_FULL) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("CSV row has more than %d columns, which is the maximum in postgres.", + MaxHeapAttributeNumber))); + else + ereport(ERROR, (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid CSV at row %lld: %s", (long long)(row_number + 1), + parsed.err.msg ? parsed.err.msg : "parse error"))); + } + + row_number++; + remaining = parsed.rest; + + size_t field_count = parsed.out.as.list.len; + + if (field_count != (size_t)vnatts) + ereport(ERROR, (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("row %lld has %zu columns, expected %d", (long long)row_number, + field_count, vnatts))); + + csv_row_to_datums(row_ctx, &arena, &parsed.out, field_count, natts, visible_attnos, infuncs, + ioparams, typmods, values, nulls); + + // The tuple values must survive long enough to be copied into the tuplestore. + MemoryContextSwitchTo(per_query_ctx); + tuplestore_putvalues(rsinfo->setResult, outdesc, values, nulls); + } + + MemoryContextDelete(row_ctx); + MemoryContextSwitchTo(oldctx); + ReleaseTupleDesc(srcdesc); + + return (Datum)0; +} diff --git a/src/pg_prelude.h b/src/pg_prelude.h index 64d9cca..6d69bfa 100644 --- a/src/pg_prelude.h +++ b/src/pg_prelude.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,7 @@ #include #include #include +#include #include #include diff --git a/test/expected/00_init.out b/test/expected/00_init.out index 5d12a2f..95451c3 100644 --- a/test/expected/00_init.out +++ b/test/expected/00_init.out @@ -23,3 +23,18 @@ CREATE TABLE nasty ( text TEXT ); INSERT INTO nasty (text) VALUES ('test'); +create table customers ( + "Index" text, + "Customer Id" text, + "First Name" text, + "Last Name" text, + "Company" text, + "City" text, + "Country" text, + "Phone 1" text, + "Phone 2" text, + "Email" text, + "Subscription Date" text, + "Website" text +); +create table empty(); diff --git a/test/expected/import.out b/test/expected/import.out new file mode 100644 index 0000000..a102986 --- /dev/null +++ b/test/expected/import.out @@ -0,0 +1,67 @@ +-- can read an inline value +select id, name +from csv_read(null::projects, E'1,IOS,4\n2,"Win""dows",4') where id = 2; + id | name +----+---------- + 2 | Win"dows +(1 row) + +-- can read a CSV file in full +select count(*) = 100 as all_read +from csv_read( + null::customers, + pg_read_file('data/customers-100.csv') +); + all_read +---------- + t +(1 row) + +-- can insert only some columns of a CSV +insert into customers +select "First Name", "Company", "Website" +from csv_read( + null::customers, + pg_read_file('data/customers-100.csv') +) +where "Index" = '4'; +\echo + +-- check inserted columns +select * from customers; + Index | Customer Id | First Name | Last Name | Company | City | Country | Phone 1 | Phone 2 | Email | Subscription Date | Website +-------+---------------------------------+----------------------------+-----------+---------+------+---------+---------+---------+-------+-------------------+--------- + Linda | Dominguez, Mcmillan and Donovan | http://www.good-lyons.com/ | | | | | | | | | +(1 row) + +-- clear inserted +truncate customers; +-- rejects NULL input +select * from csv_read(null::projects, NULL::text); +ERROR: input must not be NULL +\echo + +-- rejects empty row types +select * from csv_read(null::empty, 'id,name'); +ERROR: row type has no columns +\echo + +-- rejects rows with the wrong number of columns +select * from csv_read( + null::projects, + E'1,IOS,4\n2,Windows' +); +ERROR: row 2 has 2 columns, expected 3 +\echo + +-- rejects rows wider than PostgreSQL's column limit +select * from csv_read( + null::projects, + array_to_string(array_fill('x'::text, ARRAY[1601]), ',') +); +ERROR: CSV row has more than 1600 columns, which is the maximum in postgres. +\echo + +-- fails at parse errors +select * from csv_read(null::projects, E'1,""IOS,4'); +ERROR: invalid CSV at row 1: expected newline or end of input diff --git a/test/init.sh b/test/init.sh new file mode 100644 index 0000000..ac7e6ce --- /dev/null +++ b/test/init.sh @@ -0,0 +1,5 @@ +set -euo pipefail + +mkdir -p "$TMPDIR/data" + +ln bench/data/*.csv "$TMPDIR/data" diff --git a/test/sql/00_init.sql b/test/sql/00_init.sql index 09a0c42..ddfff00 100644 --- a/test/sql/00_init.sql +++ b/test/sql/00_init.sql @@ -26,3 +26,18 @@ CREATE TABLE nasty ( text TEXT ); INSERT INTO nasty (text) VALUES ('test'); +create table customers ( + "Index" text, + "Customer Id" text, + "First Name" text, + "Last Name" text, + "Company" text, + "City" text, + "Country" text, + "Phone 1" text, + "Phone 2" text, + "Email" text, + "Subscription Date" text, + "Website" text +); +create table empty(); diff --git a/test/sql/import.sql b/test/sql/import.sql new file mode 100644 index 0000000..eceb6f4 --- /dev/null +++ b/test/sql/import.sql @@ -0,0 +1,51 @@ +-- can read an inline value +select id, name +from csv_read(null::projects, E'1,IOS,4\n2,"Win""dows",4') where id = 2; + +-- can read a CSV file in full +select count(*) = 100 as all_read +from csv_read( + null::customers, + pg_read_file('data/customers-100.csv') +); + +-- can insert only some columns of a CSV +insert into customers +select "First Name", "Company", "Website" +from csv_read( + null::customers, + pg_read_file('data/customers-100.csv') +) +where "Index" = '4'; +\echo + +-- check inserted columns +select * from customers; + +-- clear inserted +truncate customers; + +-- rejects NULL input +select * from csv_read(null::projects, NULL::text); +\echo + +-- rejects empty row types +select * from csv_read(null::empty, 'id,name'); +\echo + +-- rejects rows with the wrong number of columns +select * from csv_read( + null::projects, + E'1,IOS,4\n2,Windows' +); +\echo + +-- rejects rows wider than PostgreSQL's column limit +select * from csv_read( + null::projects, + array_to_string(array_fill('x'::text, ARRAY[1601]), ',') +); +\echo + +-- fails at parse errors +select * from csv_read(null::projects, E'1,""IOS,4');