External Data Agent¶
This agent gathers and publishes JSON data available via a web api
Configuration¶
The following is an example configuration file for the External Data Agent:
{
#Interval at which to scrape the sources.
"interval":300,
#Global topic prefix for all publishes.
"global_topic_prefix": "record",
#Default user name and password if all sources require the same
#credentials. Can be overridden in individual sources.
#"default_user":"my_user_name",
#"default_password" : "my_password",
"sources":
[
{
#Valid types are "csv", "json", and "raw"
#Defaults to "raw"
"type": "csv",
#Source URL for CSV data.
"url": "https://example.com/example",
#URL parameters for data query (optional).
# See https://en.wikipedia.org/wiki/Query_string
"params": {"period": "currentinterval",
"format": "csv"},
#Topic to publish on.
"topic": "example/examplecsvdata1",
#Column used to break rows in CSV out into separate publishes.
#The key will be removed from the row data and appended to the end
# of the publish topic.
# If this option is missing the entire CSV will be published as a list
# of objects.
#If the column does not exist nothing will be published.
"key": "Key Column",
#Attempt to parse these columns in the data into numeric types.
#Currently columns are parsed with ast.literal_eval()
#Values that fail to parse are left as strings unless the
# values is an empty string. Empty strings are changed to None.
"parse": ["Col1", "Col2"],
#Source specific authentication.
"user":"username",
"password" : "password"
},
{
#Valid types are "csv", "json", and "raw"
#Defaults to "raw"
"type": "csv",
#Source URL for CSV data.
"url": "https://example.com/example_flat",
#URL parameters for data query (optional).
# See https://en.wikipedia.org/wiki/Query_string
"params": {"format": "csv"},
#Topic to publish on. (optional)
"topic": "example/examplecsvdata1",
#If the rows in a csv represent key/value pairs use this
#setting to reduce this format to a single object for publishing.
"flatten": true,
#Attempt to parse these columns in the data into numeric types.
#Currently columns are parsed with ast.literal_eval()
#Values that fail to parse are left as strings unless the
# values is an empty string. Empty strings are changed to None.
"parse": ["Col1", "Col2"]
},
{
#Valid types are "csv", "json", and "raw"
#Defaults to "raw"
"type": "json",
#Source URL for JSON data.
"url": "https://example.com/api/example1",
#URL parameters for data query (optional)
# See https://en.wikipedia.org/wiki/Query_string
"params": {"format": "json"},
#Topic to publish on. (optional)
"topic": "example/exampledata1",
#Path to desired data withing the JSON. Optional.
#Elements in a path may be either a string or an integer.
#Useful for peeling off unneeded layers around the wanted data.
"path": ["parentobject", "0"],
#After resolving the path above if the resulting data is a list
# the key is the path to a value in a list item. Each item in the list
# is published separately with the key appended to the end of the topic.
# Elements in a key may be a string or an integer. (optional)
"key": ["Location", "$"],
#Source specific authentication.
"user":"username",
"password" : "password"
}
]
}