csv_parse.lua
Verified Featured
Added by iNTERFACEWARE
A module for parsing well-formed CSV files.
Source Code
-- CSV Module
-- http://help.interfaceware.com/kb/parsing-csv-files
local function parseCsvLine (line,sep)
local res = {}
local pos = 1
sep = sep or ','
while true do
local c = string.sub(line,pos,pos)
if (c == "") then break end
local posn = pos
local ctest = string.sub(line,pos,pos)
trace(ctest)
while ctest == ' ' do
-- handle space(s) at the start of the line (with quoted values)
posn = posn + 1
ctest = string.sub(line,posn,posn)
if ctest == '"' then
pos = posn
c = ctest
end
end
if (c == '"') then
-- quoted value (ignore separator within)
local txt = ""
repeat
local startp,endp = string.find(line,'^%b""',pos)
txt = txt..string.sub(line,startp+1,endp-1)
pos = endp + 1
c = string.sub(line,pos,pos)
if (c == '"') then
txt = txt..'"'
-- check first char AFTER quoted string, if it is another
-- quoted string without separator, then append it
-- this is the way to "escape" the quote char in a quote. example:
-- value1,"blub""blip""boing",value3 will result in blub"blip"boing for the middle
elseif c == ' ' then
-- handle space(s) before the delimiter (with quoted values)
while c == ' ' do
pos = pos + 1
c = string.sub(line,pos,pos)
end
end
until (c ~= '"')
table.insert(res,txt)
trace(c,pos,i)
if not (c == sep or c == "") then
error("ERROR: Invalid CSV field - near character "..pos.." in this line of the CSV file: \n"..line, 3)
end
pos = pos + 1
posn = pos
ctest = string.sub(line,pos,pos)
trace(ctest)
while ctest == ' ' do
-- handle space(s) after the delimiter (with quoted values)
posn = posn + 1
ctest = string.sub(line,posn,posn)
if ctest == '"' then
pos = posn
c = ctest
end
end
else
-- no quotes used, just look for the first separator
local startp,endp = string.find(line,sep,pos)
if (startp) then
table.insert(res,string.sub(line,pos,startp-1))
pos = endp + 1
else
-- no separator found -> use rest of string and terminate
table.insert(res,string.sub(line,pos))
break
end
end
end
return res
end
------------------------------------
---- Module Interface functions ----
------------------------------------
local csv = {}
function csv.parseCsv(Data, Separator)
-- handle '\r\n\' as line separator
Data = Data:gsub('\r\n','\n')
-- handle '\r' (bad form) as line separator
Data = Data:gsub('\r','\n')
local Result={}
for Line in Data:gmatch("([^\n]+)") do
local ParsedLine = parseCsvLine(Line, Separator)
table.insert(Result, ParsedLine)
end
return Result
end
local csv_parseCsv = {
Title="csv.parseCsv";
Usage="csv.parseCsv(csv, delimiter)",
SummaryLine=[[Parses a CSV data string into a table
]];
Desc=[[Parses a CSV data string into a table.
<p> CSV is used generally to refer any format with
character delimited fields, the delimiter defautls to
a "," (comma) but you can specify any character(s)
]];
["Returns"] = {
{Desc="A table containing the parsed CSV data."},
};
ParameterTable= false,
Parameters= {
{csv= {Desc='A string containing CSV data <u>string</u>.'}},
{separator= {Desc='One or more field separator characters <u>string</u>.'}},
};
Examples={
[[ function main(Data)
local Csv = csv.parseCsv( Data) -- comma separated (default)
--local Csv = csv.parseCsv(Data, '\t')
--local Csv = csv.parseCsv(Data, '|')
trace(Csv)
end]],
};
SeeAlso={
{
Title="csv_parse.lua - in our code repository.",
Link="http://help.interfaceware.com/code/details/csv_parse-lua"
},
{
Title="CSV Parser",
Link="http://help.interfaceware.com/v6/csv-parser"
}
}
}
help.set{input_function=csv.parseCsv, help_data=csv_parseCsv}
return csv
Description
A module for parsing well-formed CSV files.
Attachments
Usage Details
This module is designed to parse well-formed CSV data. It reads CSV messages from the queue and processes them using the csv_parse.parseCsv() function.
How to use csv_parse.lua:
- Use From Translator component to read your CSV files and push the raw CSV content to the queue
- Use the csv_parse.lua module in a Filter or Destination component
- Read the CSV data from the queue and process it using the
csv_parse.parseCsv()function - Then save or forward the data as needed
Note: The attached project writes to a database, see Parsing CSV Files for details
More Information