Configuration Examples
Here are sample configuration files for different logging scenarios:
FastBCP_Settings.json : Sinks on Console + Files (Windows path style) + MSSQL
For logs with Console + Files stored on Windows machine + MSSQL sinks:
{
"ConnectionStrings": {
"MS_FastBCPLogs": "Server=localhost;Database=FastBCPLogs;Integrated Security=SSPI;Encrypt=True;TrustServerCertificate=True"
},
"Serilog": {
"Using": [
"Serilog.Sinks.Console",
"Serilog.Sinks.File",
"Serilog.Enrichers.Environment",
"Serilog.Enrichers.Thread",
"Serilog.Enrichers.Process",
"Serilog.Enrichers.Context"
],
"MinimumLevel": "Information",
"WriteTo": [
{
"Name": "Console",
"Args": {
"outputTemplate": "{Timestamp:yyyy-MM-ddTHH:mm:ss.fff zzz} -|- {Application} -|- {runid} -|- {Level:u12} -|- {fulltargetname} -|- {Message}{NewLine}{Exception}"
}
},
{
"Name": "File",
"Args": {
"fileSizeLimitBytes": 4194304,
"formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact",
"path": "Logs\\FastBCP_.json",
"retainedFileCountLimit": 10,
"rollingInterval": "Day",
"rollOnFileSizeLimit": true
}
},
{
"Name": "MSSqlServer",
"Args": {
"connectionString": "MS_FastBCPLogs",
"sinkOptionsSection": {
"TableName": "EventLogs",
"SchemaName": "dbo",
"DatabaseName": "FastTransferLogs",
"AutoCreateSqlDatabase": true,
"AutoCreateSqlTable": true,
"BatchPostingLimit": 50,
"BatchPeriod": "0.00:00:05",
"UseSqlBulkCopy": false
},
"restrictedToMinimumLevel": "Information",
"columnOptionsSection": {
"disableTriggers": true,
"clusteredColumnstoreIndex": false,
"removeStandardColumns": [
"Id",
"MessageTemplate",
"Properties",
"Exception"
],
"additionalColumns": [
{
"ColumnName": "ErrorSummary",
"PropertyName": "rootcauseMessage",
"DataType": "nvarchar",
"DataLength": 4000
},
{
"ColumnName": "EventType",
"PropertyName": "EventType",
"DataType": "varchar",
"DataLength": 50
},
{
"ColumnName": "FullTargetName",
"PropertyName": "fulltargetname",
"DataType": "nvarchar",
"DataLength": 400
},
{
"ColumnName": "Status",
"PropertyName": "finalStatus",
"DataType": "int"
},
{
"ColumnName": "Rows",
"PropertyName": "loadedRows",
"DataType": "bigint"
},
{
"ColumnName": "Columns",
"PropertyName": "loadedCols",
"DataType": "int"
},
{
"ColumnName": "ElaspedMs",
"PropertyName": "loadElapsedMs",
"DataType": "bigint"
},
{
"ColumnName": "MachineName",
"PropertyName": "MachineName",
"DataType": "varchar",
"DataLength": 256
},
{
"ColumnName": "Application",
"PropertyName": "Application",
"DataType": "varchar",
"DataLength": 50
},
{
"ColumnName": "ProcessId",
"PropertyName": "ProcessId",
"DataType": "int"
},
{
"ColumnName": "ThreadId",
"PropertyName": "ThreadId",
"DataType": "int"
},
{
"ColumnName": "RunId",
"PropertyName": "runid",
"nonClusteredIndex": true,
"DataType": "varchar",
"DataLength": 256
},
{
"ColumnName": "TraceId",
"PropertyName": "traceid",
"nonClusteredIndex": true,
"DataType": "varchar",
"DataLength": 100
},
{
"ColumnName": "SpanId",
"PropertyName": "SpanId",
"DataType": "varchar",
"DataLength": 100
},
{
"ColumnName": "Release",
"PropertyName": "release",
"DataType": "varchar",
"DataLength": 32
},
{
"ColumnName": "EnvironmentUserName",
"PropertyName": "EnvironmentUserName",
"DataType": "varchar",
"DataLength": 200
},
{
"ColumnName": "InputParameters",
"PropertyName": "InputParameters",
"DataType": "varchar",
"DataLength": 8000
}
],
"level": {
"columnName": "Severity",
"storeAsEnum": false,
"DataType": "varchar",
"DataLength": 50
},
"timeStamp": {
"columnName": "EventTimestamp",
"nonClusteredIndex": true,
"convertToUtc": false
},
"logEvent": {
"excludeAdditionalProperties": false,
"excludeStandardColumns": true
},
"message": {
"columnName": "EventMsg",
"DataType": "varchar",
"DataLength": 8000
}
}
}
}
],
"Enrich": [
"FromLogContext",
"WithMachineName",
"WithProcessId",
"WithThreadId"
],
"Properties": {
"Application": "FastBCP"
}
}
}
FastBCP_Settings_Linux_Console_Files.json
For Linux with Console + Daily rolling Files:
{
"Serilog": {
"Using": [
"Serilog.Sinks.Console",
"Serilog.Sinks.File",
"Serilog.Enrichers.Environment",
"Serilog.Enrichers.Thread",
"Serilog.Enrichers.Process",
"Serilog.Enrichers.Context"
],
"MinimumLevel": "Information",
"WriteTo": [
{
"Name": "Console",
"Args": {
"outputTemplate": "{Timestamp:yyyy-MM-ddTHH:mm:ss.fff zzz} -|- {Application} -|- {runid} -|- {Level:u12} -|- {fulltargetname} -|- {Message}{NewLine}{Exception}"
}
},
{
"Name": "File",
"Args": {
"fileSizeLimitBytes": 4194304,
"formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact",
"path": "Logs/FastBCP_.json",
"retainedFileCountLimit": 10,
"rollingInterval": "Day",
"rollOnFileSizeLimit": true
}
}
],
"Enrich": [
"FromLogContext",
"WithMachineName",
"WithProcessId",
"WithThreadId"
],
"Properties": {
"Application": "FastBCP"
}
}
}
FastBCP_Settings_Map_for_Windows.json
For Console and dynamically named log file targets (Windows):
{
"Serilog": {
"Using": [
"Serilog.Sinks.Console",
"Serilog.Sinks.File",
"Serilog.Enrichers.Environment",
"Serilog.Enrichers.Thread",
"Serilog.Enrichers.Process",
"Serilog.Enrichers.Context"
],
"MinimumLevel": "Information",
"WriteTo": [
{
"Name": "Console",
"Args": {
"outputTemplate": "{Timestamp:yyyy-MM-ddTHH:mm:ss.fff zzz} -|- {Application} -|- {runid} -|- {Level:u12} -|- {fulltargetname} -|- {Message}{NewLine}{Exception}"
}
},
{
"Name": "Map",
"Args": {
"keyPropertyName": "TraceId",
"defaultKey": "no-trace",
"to": [
{
"Name": "File",
"Args": {
"path": "C:\\temp\\Logs\\{logdate}\\{sourcedatabase}\\{sourcetable}\\FastBCP-log-{filename}-{logtimestamp}-{traceid}.json",
"formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact",
"rollingInterval": "Infinite",
"shared": false,
"encoding": "utf-8",
"retainedFileCountLimit": 100
}
}
]
}
}
],
"Enrich": [
"FromLogContext",
"WithMachineName",
"WithProcessId",
"WithThreadId"
],
"Properties": {
"Application": "FastBCP"
}
}
}
FastBCP_Settings_Map_for_Linux_and_Container.json
For Console and dynamically named log file targets (Linux/Container):
{
"Serilog": {
"Using": [
"Serilog.Sinks.Console",
"Serilog.Sinks.File",
"Serilog.Enrichers.Environment",
"Serilog.Enrichers.Thread",
"Serilog.Enrichers.Process",
"Serilog.Enrichers.Context"
],
"MinimumLevel": "Information",
"WriteTo": [
{
"Name": "Console",
"Args": {
"outputTemplate": "{Timestamp:yyyy-MM-ddTHH:mm:ss.fff zzz} -|- {Application} -|- {runid} -|- {Level:u12} -|- {fulltargetname} -|- {Message}{NewLine}{Exception}"
}
},
{
"Name": "Map",
"Args": {
"keyPropertyName": "TraceId",
"defaultKey": "no-trace",
"to": [
{
"Name": "File",
"Args": {
"path": "/logs/{logdate}/{sourcedatabase}/{sourcetable}/FastBCP-log-{filename}-{logtimestamp}-{traceid}.json",
"formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact",
"rollingInterval": "Infinite",
"shared": false,
"encoding": "utf-8",
"retainedFileCountLimit": 100
}
}
]
}
}
],
"Enrich": [
"FromLogContext",
"WithMachineName",
"WithProcessId",
"WithThreadId"
],
"Properties": {
"Application": "FastBCP"
}
}
}
FastBCP_Settings_Log_to_PGSQL.json : Console + PostgreSQL log database
{
"ConnectionStrings": {
"PG_FastBCPLogs": "Host=localhost;Port=15432;Database=fastlogs;Username=FastUser;Password=FastPassword"
},
"Serilog": {
"Using": [
"Serilog.Sinks.Console",
"Serilog.Sinks.PostgreSQL.Alternative",
"Serilog.Enrichers.Environment",
"Serilog.Enrichers.Thread",
"Serilog.Enrichers.Process",
"Serilog.Enrichers.Context"
],
"MinimumLevel": "Information",
"WriteTo": [
{
"Name": "Console",
"Args": {
"outputTemplate": "{Timestamp:yyyy-MM-ddTHH:mm:ss.fff zzz} -|- {Application} -|- {runid} -|- {Level:u12} -|- {fulltargetname} -|- {Message}{NewLine}{Exception}"
}
},
{
"Name": "PostgreSQL",
"Args": {
"connectionString": "PG_FastBCPLogs",
"tableName": "event_logs",
"schemaName": "public",
"needAutoCreateTable": true,
"needAutoCreateSchema": false,
"restrictedToMinimumLevel": "Information",
"useCopy": false,
"loggerColumnOptions": {
"event_timestamp": {
"Name": "Timestamp",
"Format": "",
"WriteMethod": "Timestamp",
"NpgsqlDbType": "TimestampTz",
"Order": 1
},
"event_msg": {
"Name": "RenderedMessage",
"Format": "",
"WriteMethod": "RenderedMessage",
"NpgsqlDbType": "Text",
"Order": 2
},
"Level": {
"Name": "LevelAsText",
"Order": 3
},
"LevelInt": {
"Name": "Level",
"Order": 4
}
},
"loggerPropertyColumnOptions": {
"error_summary": {
"Name": "rootcauseMessage",
"Format": "{0}",
"WriteMethod": "ToString",
"NpgsqlDbType": "Text",
"Order": 5
},
"source_database": {
"Name": "SourceDatabase",
"WriteMethod": "Raw",
"NpgsqlDbType": "Text",
"Order": 6
},
"source_schema": {
"Name": "SourceSchema",
"WriteMethod": "Raw",
"NpgsqlDbType": "Text",
"Order": 7
},
"source_table": {
"Name": "SourceTable",
"WriteMethod": "Raw",
"NpgsqlDbType": "Text",
"Order": 8
},
"output_directory": {
"Name": "OutputDirectory",
"WriteMethod": "Raw",
"NpgsqlDbType": "Text",
"Order": 9
},
"full_target_name": {
"Name": "fulltargetname",
"WriteMethod": "Raw",
"NpgsqlDbType": "Text",
"Order": 10
},
"application": {
"Name": "Application",
"WriteMethod": "Raw",
"NpgsqlDbType": "Text",
"Order": 11
},
"status": {
"Name": "finalStatus",
"Format": "0",
"WriteMethod": "ToString",
"NpgsqlDbType": "Integer",
"Order": 12
},
"rows": {
"Name": "loadedRows",
"Format": "0",
"WriteMethod": "ToString",
"NpgsqlDbType": "Numeric",
"Order": 13
},
"columns": {
"Name": "loadedCols",
"Format": "0",
"WriteMethod": "ToString",
"NpgsqlDbType": "Integer",
"Order": 14
},
"elapsed_ms": {
"Name": "loadElapsedMs",
"Format": "0",
"WriteMethod": "ToString",
"NpgsqlDbType": "Numeric",
"Order": 15
},
"machine_name": {
"Name": "MachineName",
"WriteMethod": "Raw",
"NpgsqlDbType": "Text",
"Order": 16
},
"process_id": {
"Name": "ProcessId",
"Format": "0",
"WriteMethod": "ToString",
"NpgsqlDbType": "Integer",
"Order": 17
},
"thread_id": {
"Name": "ThreadId",
"Format": "0",
"WriteMethod": "ToString",
"NpgsqlDbType": "Integer",
"Order": 18
},
"run_id": {
"Name": "runid",
"WriteMethod": "Raw",
"NpgsqlDbType": "Text",
"Order": 19
},
"trace_id": {
"Name": "traceid",
"WriteMethod": "Raw",
"NpgsqlDbType": "Text",
"Order": 20
},
"span_id": {
"Name": "SpanId",
"WriteMethod": "Raw",
"NpgsqlDbType": "Text",
"Order": 21
},
"release": {
"Name": "release",
"WriteMethod": "Raw",
"NpgsqlDbType": "Text",
"Order": 22
},
"environment_user_name": {
"Name": "EnvironmentUserName",
"Format": "",
"WriteMethod": "Raw",
"NpgsqlDbType": "Text",
"Order": 23
},
"input_parameters": {
"Name": "InputParameters",
"WriteMethod": "Raw",
"NpgsqlDbType": "Json",
"Order": 24
}
}
}
}
],
"Enrich": [
"FromLogContext",
"WithMachineName",
"WithProcessId",
"WithThreadId"
],
"Properties": {
"Application": "FastBCP"
}
}
}
FastBCP_Settings_OpenTelemetry.json
For Console with trace correlation and OpenTelemetry log export (e.g., to Jaeger):
{
"Serilog": {
"MinimumLevel": {
"Default": "Debug",
"Override": {
"Microsoft": "Warning",
"System": "Warning"
}
},
"Properties": {
"LogDirectory": "logs"
},
"Enrich": [
"FromLogContext",
"WithEnvironmentUserName",
"WithProcessId",
"WithThreadId",
"WithMachineName",
"fulltargetname"
],
"WriteTo": [
{
"Name": "Console",
"Args": {
"restrictedToMinimumLevel": "Information",
"outputTemplate": "{Timestamp:yyyy-MM-ddTHH:mm:ss.fff zzz} -|- {Application} -|- {Level:u12} -|- {fulltargetname} -|- TraceId={TraceId} SpanId={SpanId} -|- {Message}{NewLine}{Exception}",
"theme": "Serilog.Sinks.SystemConsole.Themes.SystemConsoleTheme::Literate, Serilog.Sinks.Console"
}
},
{
"Name": "OpenTelemetry",
"Args": {
"Endpoint": "http://localhost:4317",
"Protocol": "Grpc",
"ResourceAttributes": {
"service.name": "FastBCP",
"deployment.environment": "local-test"
}
}
}
]
}
}
Usage with the --traceparent parameter to correlate with an upstream caller:
./FastBCP \
--connectiontype mssql \
--server localhost \
--database "tpch10" \
--trusted \
--query "SELECT * FROM dbo.orders" \
--directory "/data/temp" \
--fileoutput "orders.csv" \
--traceparent "00-abcdef1234567890abcdef1234567890-1234567890abcdef-01" \
--settingsfile "/config/FastBCP_Settings_OpenTelemetry.json"
Result in Jaeger UI:
