{
  "$id": "https://json.schemastore.org/starlake.json",
  "$schema": "http://json-schema.org/draft-07/schema#",
  "definitions": {
    "PrimitiveType": {
      "description": "Default types defined in starlake. User defined types may also be defined in the types/types.sl.yml file",
      "type": "string",
      "oneOf": [
        {
          "const": "string",
          "description": "Any string that match the '.*' regex"
        },
        {
          "const": "long",
          "description": "Any whole number that match the '[-|+|0-9][0-9]*' regex.\nints are mapped to as 'int' in some database whereas 'longs are mapped as 'bigint' and shorts as 'smallint'"
        },
        {
          "const": "int",
          "description": "Any whole number that match the '[-|+|0-9][0-9]*' regex.\nints are mapped to as 'int' in some database whereas 'longs are mapped as 'bigint' and shorts as 'smallint'"
        },
        {
          "const": "short",
          "description": "Any whole number that match the '[-|+|0-9][0-9]*' regex.\nints are mapped to as 'int' in some database whereas 'longs are mapped as 'bigint' and shorts as 'smallint'"
        },
        {
          "const": "double",
          "description": "Any decimal number that match the '[-+]?\\d*\\.?\\d+[Ee]?[-+]?\\d*' regex"
        },
        {
          "const": "boolean",
          "description": "Any string that match the '(?i)true|yes|[y1]<-TF->(?i)false|no|[n0]' regex,\nwhere the value on the left of '<-T' represent true and values on the right of 'F->' represent the false"
        },
        {
          "const": "byte",
          "description": "Any single char"
        },
        {
          "const": "date",
          "description": "Any date that match the 'yyyy-MM-dd' regex (2023-12-31)"
        },
        {
          "const": "basic_iso_date",
          "description": "Any date that match the 'yyyyMMdd' regex (20231231)"
        },
        {
          "const": "iso_local_date",
          "description": "Any date that match the 'yyyy-MM-dd' regex (2023-12-31)"
        },
        {
          "const": "iso_offset_date",
          "description": "Timestamp based on `ISO Date with offset` pattern (2023-12-31+02:00)"
        },
        {
          "const": "iso_date",
          "description": "Timestamp based on `ISO Date with or without offset` pattern (2011-12-03+02:00)"
        },
        {
          "const": "iso_local_date_time",
          "description": "Timestamp based on `ISO Local Date and Time` pattern (2023-12-31T10:15:30)"
        },
        {
          "const": "iso_offset_date_time",
          "description": "Timestamp based on `ISO Local Date and Time` pattern (2023-12-31T10:15:30+01:00)"
        },
        {
          "const": "iso_zoned_date_time",
          "description": "Timestamp based on `ISO Zoned Date Time`pattern (2023-12-31T10:15:30+01:00[Europe/Paris])"
        },
        {
          "const": "iso_date_time",
          "description": "Timestamp based on `ISO Date and time with ZoneId` pattern (2011-12-03T10:15:30+01:00[Europe/Paris])"
        },
        {
          "const": "iso_ordinal_date",
          "description": "Timestamp based on `year and day of year` pattern (2023-337)"
        },
        {
          "const": "iso_week_date",
          "description": "Timestamp based on `Year and Week` pattern (2012-W48-6)"
        },
        {
          "const": "iso_instant",
          "description": "Timestamp based on `Date and Time of an Instant` pattern - UTC only - (2023-12-31T10:15:30Z)"
        },
        {
          "const": "rfc_1123_date_time",
          "description": "Timestamp based on `RFC 1123 / RFC 822` patterns (Tue, 3 Jun 2008 11:05:30 GMT)"
        },
        {
          "const": "timestamp",
          "description": "date/time that match the 'yyyy-MM-dd HH:mm:ss' regex s (2019-12-31 23:59:02).\nFor epoch timestamp, set pattern attribute to 'epoch_second' or 'epoch_milli'"
        },
        {
          "const": "decimal",
          "description": "Any floating value that match the '-?\\d*\\.{0,1}\\d+' regex"
        },
        {
          "const": "struct",
          "description": "Any attribute that has children. Set the array to true if this attribute is made of a list of attributes"
        }
      ]
    },
    "IndexMapping": {
      "type": "string",
      "oneOf": [
        {
          "const": "text",
          "description": ""
        },
        {
          "const": "keyword",
          "description": ""
        },
        {
          "const": "long",
          "description": ""
        },
        {
          "const": "integer",
          "description": ""
        },
        {
          "const": "short",
          "description": ""
        },
        {
          "const": "byte",
          "description": ""
        },
        {
          "const": "double",
          "description": ""
        },
        {
          "const": "float",
          "description": ""
        },
        {
          "const": "half_float",
          "description": ""
        },
        {
          "const": "scaled_float",
          "description": ""
        },
        {
          "const": "date",
          "description": ""
        },
        {
          "const": "boolean",
          "description": ""
        },
        {
          "const": "binary",
          "description": ""
        },
        {
          "const": "integer_rang",
          "description": ""
        },
        {
          "const": "float_range",
          "description": ""
        },
        {
          "const": "long_range",
          "description": ""
        },
        {
          "const": "double_range",
          "description": ""
        },
        {
          "const": "date_range",
          "description": ""
        },
        {
          "const": "geo_point",
          "description": ""
        },
        {
          "const": "geo_shape",
          "description": ""
        },
        {
          "const": "ip",
          "description": ""
        },
        {
          "const": "completion",
          "description": ""
        },
        {
          "const": "token_count",
          "description": ""
        },
        {
          "const": "object",
          "description": ""
        },
        {
          "const": "array",
          "description": ""
        }
      ]
    },
    "WriteMode": {
      "description": "How the data is written to the target datawarehouse. Default is APPEND",
      "type": "string",
      "oneOf": [
        {
          "const": "OVERWRITE",
          "description": "That data will overwrite the existing data or create it if it does not exist"
        },
        {
          "const": "APPEND",
          "description": "Append the data to an existing table or create it if it does not exist"
        },
        {
          "const": "ERROR_IF_EXISTS",
          "description": "Fail if the table already exist"
        },
        {
          "const": "IGNORE",
          "description": "Do not save at all. Useful in interactive / test mode."
        }
      ]
    },
    "UserType": {
      "description": "User types defined in starlake. Used as a prefix in row level and column level security policies",
      "type": "string",
      "oneOf": [
        {
          "const": "SA",
          "description": "Service account"
        },
        {
          "const": "USER",
          "description": "End user"
        },
        {
          "const": "GROUP",
          "description": "Group of users / service accounts"
        }
      ]
    },
    "Trim": {
      "description": "How to trim the input string",
      "type": "string",
      "oneOf": [
        {
          "const": "LEFT",
          "description": "Remove all leading space chars from the input"
        },
        {
          "const": "RIGHT",
          "description": "Remove all trailing spaces from the input"
        },
        {
          "const": "BOTH",
          "description": "Remove all leading and trailing spaces from the input"
        },
        {
          "const": "NONE",
          "description": "Do not remove leading or trailing spaces from the input"
        }
      ]
    },
    "TableDdl": {
      "description": "DDL used to create a table",
      "type": "object",
      "properties": {
        "createSql": {
          "type": "string",
          "description": "SQL CREATE DDL statement"
        },
        "pingSql": {
          "type": "string",
          "description": "How to test if the table exist.\nUse the following statement by default: 'select count(*) from tableName where 1=0'"
        }
      },
      "required": ["createSql"]
    },
    "TableType": {
      "type": "string",
      "description": "Table types supported by the Extract module",
      "oneOf": [
        {
          "const": "TABLE",
          "description": "SQl Table"
        },
        {
          "const": "VIEW",
          "description": "SQl View"
        },
        {
          "const": "SYSTEM TABLE",
          "description": "Database specific system table"
        },
        {
          "const": "GLOBAL TEMPORARY",
          "description": ""
        },
        {
          "const": "LOCAL TEMPORARY",
          "description": ""
        },
        {
          "const": "ALIAS",
          "description": "Table alias"
        },
        {
          "const": "SYNONYM",
          "description": "Table synonym"
        }
      ]
    },
    "Type": {
      "type": "object",
      "description": "Custom type definition. Custom types are defined in the types/types.sl.yml file",
      "properties": {
        "name": {
          "type": "string",
          "description": "unique id for this type"
        },
        "primitiveType": {
          "$ref": "#/definitions/PrimitiveType",
          "description": "To what primitive type should this type be mapped.\n This is the memory representation of the type, When saving, this primitive type is mapped to the database specific type  "
        },
        "pattern": {
          "type": "string",
          "description": "Regex used to validate the input field"
        },
        "zone": {
          "type": "string",
          "description": "useful when parsing specific string:\n - double: To parse a french decimal (comma as decimal separator) set it to fr_FR locale.\n- decimal: to set the precision and scale of this number, '38,9' by default.\n- "
        },
        "sample": {
          "type": "string",
          "description": "This field makes sure that the pattern matches the value you want to match. This will be checked on startup"
        },
        "comment": {
          "type": "string",
          "description": "Describes this type"
        },
        "indexMapping": {
          "type": "string",
          "description": "How this type is indexed in your datawarehouse"
        },
        "ddlMapping": {
          "$ref": "#/definitions/MapString",
          "description": "Configure here the type mapping for each datawarehouse.\\nWill be used when inferring DDL from schema."
        }
      },
      "required": ["name", "pattern", "primitiveType"]
    },
    "Partition": {
      "description": "Partition columns, no partitioning by default",
      "type": "object",
      "properties": {
        "sampling": {
          "type": "number",
          "description": "0.0 means no sampling, > 0  && < 1 means sample dataset, >=1 absolute number of partitions. Used exclusively on Hadoop like warehouses"
        },
        "attributes": {
          "type": "array",
          "items": {
            "type": "string",
            "description": "Attributes used to partition de dataset."
          }
        }
      },
      "required": []
    },
    "Position": {
      "description": "First and last char positions of an attribute in a fixed length record",
      "type": "object",
      "properties": {
        "first": {
          "type": "number",
          "description": "Zero based position of the first character for this attribute"
        },
        "last": {
          "type": "number",
          "description": "Zero based position of the last character to include in this attribute"
        }
      },
      "required": ["first", "last"]
    },
    "ExpectationItem": {
      "description": "Expectation",
      "type": "object",
      "properties": {
        "query": {
          "type": "string",
          "description": "SQL query to execute"
        },
        "expect": {
          "type": "string",
          "description": "What outcome is expected"
        }
      },
      "required": ["query", "expect"]
    },
    "Connection": {
      "description": "Connection properties to a datawarehouse.",
      "type": "object",
      "properties": {
        "type": {
          "type": "string",
          "description": "aka jdbc, bigquery, snowflake, redshift ..."
        },
        "sparkFormat": {
          "type": "string",
          "description": "Set only if you want to use the Spark engine"
        },
        "options": {
          "$ref": "#/definitions/MapString",
          "description": "Connection options"
        }
      },
      "required": ["type"]
    },
    "DagGenerationConfig": {
      "description": "Dag configuration.",
      "type": "object",
      "properties": {
        "comment": {
          "type": "string",
          "description": "Dag config description"
        },
        "template": {
          "type": "string",
          "description": "Dag template to use for this config. Usually a .py.j2 file"
        },
        "filnename": {
          "type": "string",
          "description": "{schedule}, {domain}, {table} in the file name are used for DAG generation purposes"
        },
        "options": {
          "$ref": "#/definitions/MapString",
          "description": "DAG generation options"
        }
      },
      "required": ["type"]
    },
    "RowLevelSecurity": {
      "description": "Row level security policy to apply to the output data.",
      "type": "object",
      "properties": {
        "name": {
          "type": "string",
          "description": "This Row Level Security unique name"
        },
        "description": {
          "type": "string",
          "description": "Description for this access policy"
        },
        "predicate": {
          "type": "string",
          "description": "The condition that goes to the WHERE clause and limit the visible rows."
        },
        "grants": {
          "description": "user / groups / service accounts to which this security level is applied.\nex : user:me@mycompany.com,group:group@mycompany.com,serviceAccount:mysa@google-accounts.com",
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      },
      "required": ["name", "grants"]
    },
    "AccessControlEntry": {
      "description": "Column level security policy to apply to the attribute.",
      "type": "object",
      "properties": {
        "role": {
          "type": "string",
          "description": "This role to give to the granted users"
        },
        "grants": {
          "description": "user / groups / service accounts to which this security level is applied.\nex : user:me@mycompany.com,group:group@mycompany.com,serviceAccount:mysa@google-accounts.com",
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      },
      "required": ["role", "grants"]
    },
    "MergeOptions": {
      "type": "object",
      "properties": {
        "key": {
          "description": "list of attributes to join an existing and incoming dataset. Use renamed columns if any here.",
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "timestamp": {
          "type": "string",
          "description": "Timestamp column used to identify last version, if not specified currently ingested row is considered the last"
        },
        "queryFilter": {
          "type": "string",
          "description": "Useful when you want to merge only on a subset of the existing partitions, thus improving performance and reducing costs.\nYou may use here:\n- Any SQL condition\n- latest which will be translated to the last existing partition\n- column in last(10) which will apply the merge on the last 10 partitions of your dataset.\n last and latest assume that your table is partitioned by day."
        }
      },
      "required": ["key"]
    },
    "Format": {
      "description": "DSV by default. Supported file formats are :\\n- DSV : Delimiter-separated values file. Delimiter value is specified in the \"separator\" field.\\n- POSITION : FIXED format file where values are located at an exact position in each line.\\n- SIMPLE_JSON : For optimisation purpose, we differentiate JSON with top level values from JSON\\n  with deep level fields. SIMPLE_JSON are JSON files with top level fields only.\\n- JSON :  Deep JSON file. Use only when your json documents contain sub-documents, otherwise prefer to\\n  use SIMPLE_JSON since it is much faster.\\n- XML : XML files",
      "type": "string",
      "oneOf": [
        {
          "const": "DSV",
          "description": "any single or multiple character delimited file. Separator is specified in the separator field"
        },
        {
          "const": "POSITION",
          "description": "any fixed position file. Positions are specified in the position field"
        },
        {
          "const": "JSON",
          "description": "any deep json file.\nTo improve performance, prefer the SIMPLE_JSON format if your json documents are flat"
        },
        {
          "const": "ARRAY_JSON",
          "description": "any json file containing an array of json objects."
        },
        {
          "const": "SIMPLE_JSON",
          "description": "any flat json file.\nTo improve performance, prefer this format if your json documents are flat"
        },
        {
          "const": "XML",
          "description": "any xml file. Use the metadata.xml.rowTag field to specify the root tag of your xml file"
        }
      ]
    },
    "MapString": {
      "type": "object",
      "description": "Map of string",
      "additionalProperties": {
        "type": "string"
      }
    },
    "MapConnection": {
      "type": "object",
      "description": "Map of connections",
      "additionalProperties": {
        "$ref": "#/definitions/Connection"
      }
    },
    "MapExpectationItem": {
      "type": "object",
      "description": "Map of connections",
      "additionalProperties": {
        "$ref": "#/definitions/ExpectationItem"
      }
    },
    "MapJdbcEngine": {
      "type": "object",
      "description": "Map of jdbc engines",
      "additionalProperties": {
        "$ref": "#/definitions/JdbcEngine"
      }
    },
    "MapTableDdl": {
      "type": "object",
      "description": "Map of table ddl",
      "additionalProperties": {
        "$ref": "#/definitions/TableDdl"
      }
    },
    "JdbcEngine": {
      "type": "object",
      "description": "Jdbc engine",
      "properties": {
        "tables": {
          "description": "List of all SQL create statements used to create audit tables for this JDBC engine.\nTables are created only if the execution of the pingSQL statement fails",
          "type": "array",
          "items": {
            "$ref": "#/definitions/MapTableDdl"
          }
        }
      }
    },
    "Privacy": {
      "type": "object",
      "properties": {
        "options": {
          "description": "Privacy strategies. The following default strategies are defined by default:\n- none: Leave the data as is\n- hide: replace the data with an empty string\n- hideX(\"s\", n): replace the string with n occurrences of the string 's'\n- md5: Redact the data using the MD5 algorithm\n- sha1: Redact the data using the SHA1 algorithm\n- sha256: Redact the data using the SHA256 algorithm\n - sha512: Redact the data using the SHA512 algorithm\n- initials: keep only the first char of each word in the data",
          "$ref": "#/definitions/MapString"
        }
      }
    },
    "Internal": {
      "type": "object",
      "description": "configure Spark internal options",
      "properties": {
        "cacheStorageLevel": {
          "description": "How the RDD are cached. Default is MEMORY_AND_DISK_SER.\nAvailable options are (https://spark.apache.org/docs/latest/api/java/index.html?org/apache/spark/storage/StorageLevel.html):\n- MEMORY_ONLY\n- MEMORY_AND_DISK\n- MEMORY_ONLY_SER\n- MEMORY_AND_DISK_SER\n- DISK_ONLY\n- OFF_HEAP",
          "type": "string"
        },
        "intermediateBigqueryFormat": {
          "description": "May be parquet or ORC. Default is parquet. Used for BigQuery intermediate storage. Use ORC for for JSON files to keep the original data structure.\nhttps://stackoverflow.com/questions/53674838/spark-writing-parquet-arraystring-converts-to-a-different-datatype-when-loadin",
          "type": "string"
        },
        "temporaryGcsBucket": {
          "description": "The GCS bucket that temporarily holds the data before it is loaded to BigQuery.",
          "type": "string"
        },
        "substituteVars": {
          "description": "Internal use. Do not modify.",
          "type": "boolean"
        }
      }
    },
    "AccessPolicies": {
      "type": "object",
      "properties": {
        "apply": {
          "description": "Should access policies be enforced ?",
          "type": "boolean"
        },
        "location": {
          "description": "GCP project location. Required if apply is true.",
          "type": "string"
        },
        "database": {
          "description": "GCP Project id. Required if apply is true.",
          "type": "string"
        },
        "taxonomy": {
          "description": "Taxonomy name. Required if apply is true.",
          "type": "string"
        }
      }
    },
    "JobScheduling": {
      "type": "object",
      "properties": {
        "maxJobs": {
          "description": "Max number of Spark jobs to run in parallel, default is 1",
          "type": "integer"
        },
        "poolName": {
          "description": "Pool name to use for Spark jobs, default is 'default'",
          "type": "string"
        },
        "mode": {
          "description": "This can be FIFO or FAIR, to control whether jobs within the pool queue up behind each other (the default) or share the pool’s resources fairly.",
          "type": "string"
        },
        "file": {
          "description": "Scheduler filename in the metadata folder. If not set, defaults to fairscheduler.xml.",
          "type": "string"
        }
      }
    },
    "ExpectationsConfig": {
      "type": "object",
      "properties": {
        "path": {
          "description": "When using filesystem storage, the path to the expectations file",
          "type": "string"
        },
        "active": {
          "description": "should expectations be executed ?",
          "type": "boolean"
        },
        "failOnError": {
          "description": "should load / transform fail on expectation error ?",
          "type": "boolean"
        }
      }
    },
    "Metrics": {
      "type": "object",
      "properties": {
        "path": {
          "description": "When using filesystem storage, the path to the metrics file",
          "type": "string"
        },
        "discreteMaxCardinality": {
          "description": "Max number of unique values accepted for a discrete column. Default is 10",
          "type": "integer"
        },
        "active": {
          "description": "Should metrics be computed ?",
          "type": "boolean"
        }
      }
    },
    "Sink": {
      "type": "object",
      "properties": {
        "options": {
          "$ref": "#/definitions/MapString",
          "description": "spark  options to use"
        },
        "id": {
          "type": "string",
          "description": "ES: Attribute to use as id of the document. Generated by Elasticsearch if not specified."
        },
        "timestamp": {
          "type": "string",
          "description": "ES or BQ: The timestamp column to use for table partitioning if any. No partitioning by default\\nES:Timestamp field format as expected by Elasticsearch (\"{beginTs|yyyy.MM.dd}\" for example)."
        },
        "location": {
          "type": "string",
          "description": "BQ: Database location (EU, US, ...)"
        },
        "clustering": {
          "description": "FS or BQ: List of attributes to use for clustering",
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "days": {
          "type": "number",
          "description": "BQ: Number of days before this table is set as expired and deleted. Never by default."
        },
        "requirePartitionFilter": {
          "type": "boolean",
          "description": "BQ: Should be require a partition filter on every request ? No by default."
        },
        "materializedView": {
          "type": "boolean",
          "description": "BQ: Should we materialize as a table or as a view when saving the results ? false by default."
        },
        "enableRefresh": {
          "type": "boolean",
          "description": "BQ: Enable automatic refresh of materialized view ? false by default."
        },
        "refreshIntervalMs": {
          "type": "number",
          "description": "BQ: Refresh interval in milliseconds. Default to BigQuery default value"
        },
        "format": {
          "type": "string",
          "description": "FS: File format"
        },
        "extension": {
          "type": "string",
          "description": "FS: File extension"
        },
        "partition": {
          "$ref": "#/definitions/Partition",
          "description": "FS or BQ: List of partition attributes"
        },
        "connectionRef": {
          "type": "string",
          "description": "JDBC: Connection String"
        },
        "coalesce": {
          "type": "boolean",
          "description": "When outputting files, should we coalesce it to a single file. Useful when CSV is the output format."
        },
        "dynamicPartitionOverwrite": {
          "type": "boolean",
          "description": "When outputting files, should we overwrite existing partitions ?"
        }
      }
    },
    "MapArrayOfString": {
      "type": "object",
      "additionalProperties": {
        "type": "array",
        "items": {
          "type": "string"
        }
      }
    },
    "Metadata": {
      "type": "object",
      "properties": {
        "format": {
          "$ref": "#/definitions/Format"
        },
        "encoding": {
          "type": "string",
          "description": "UTF-8 if not specified."
        },
        "multiline": {
          "type": "boolean",
          "description": "are json objects on a single line or multiple line ? Single by default.  false means single. false also means faster"
        },
        "array": {
          "type": "boolean",
          "description": "Is the json stored as a single object array ? false by default. This means that by default we have on json document per line."
        },
        "withHeader": {
          "type": "boolean",
          "description": "does the dataset has a header ? true by default"
        },
        "separator": {
          "type": "string",
          "description": "the values delimiter,  ';' by default value may be a multichar string starting from Spark3"
        },
        "quote": {
          "type": "string",
          "description": "The String quote char, '\"' by default"
        },
        "escape": {
          "type": "string",
          "description": "escaping char '\\' by default"
        },
        "write": {
          "$ref": "#/definitions/WriteMode",
          "description": "Write mode, APPEND by default"
        },
        "partition": {
          "$ref": "#/definitions/Partition"
        },
        "sink": {
          "$ref": "#/definitions/Sink"
        },
        "ignore": {
          "type": "string",
          "description": "Pattern to ignore or UDF to apply to ignore some lines"
        },
        "xml": {
          "$ref": "#/definitions/MapString",
          "description": "com.databricks.spark.xml options to use (eq. rowTag)"
        },
        "directory": {
          "description": "Folder on the local filesystem where incoming files are stored.\n Typically, this folder will be scanned periodically to move the dataset to the cluster for ingestion.\n                     Files located in this folder are moved to the pending folder for ingestion by the \"import\" command.",
          "type": "string"
        },
        "extensions": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "recognized filename extensions. json, csv, dsv, psv are recognized by default.\nOnly files with these extensions will be moved to the pending folder."
        },
        "ack": {
          "description": "Ack extension used for each file. \".ack\" if not specified.\nFiles are moved to the pending folder only once a file with the same name as the source file and with this extension is present.\nTo move a file without requiring an ack file to be present, set explicitly this property to the empty string value \"\".",
          "type": "string"
        },
        "options": {
          "$ref": "#/definitions/MapString",
          "description": "Options to add to the spark reader"
        },
        "loader": {
          "description": "Loader to use, 'spark' or 'native'. Default to 'spark' of SL_LOADER env variable is set to 'native'",
          "type": "string"
        },
        "emptyIsNull": {
          "description": "Treat empty columns as null in DSV files. Default to false",
          "type": "boolean"
        },
        "nullValue": {
          "description": "Treat a specific input string as a null value indicator",
          "type": "string"
        },
        "freshness": {
          "description": "Configure freshness checks on this dataset",
          "$ref": "#/definitions/Freshness"
        },
        "schedule": {
          "type": "string",
          "description": "Cron expression to use for this domain/table"
        },
        "dagRef": {
          "type": "string",
          "description": "Cron expression to use for this domain/table"
        }
      }
    },
    "ExternalDomain": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string",
          "description": "Schema in JDBC Database / Snowflake / Redshift or Dataset in BigQuery"
        },
        "tables": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "Tables to scan in this domain"
        }
      }
    },
    "External": {
      "type": "object",
      "properties": {
        "database": {
          "type": "string",
          "description": "Database name of Project id in BigQuery"
        },
        "external": {
          "description": "List of domains to scan",
          "type": "array",
          "items": {
            "$ref": "#/definitions/ExternalDomain"
          }
        }
      }
    },
    "Area": {
      "type": "object",
      "properties": {
        "pending": {
          "type": "string",
          "description": "Files recognized by the extensions property are moved to this folder for ingestion by the \"import\" command."
        },
        "unresolved": {
          "description": "Files that cannot be ingested (do not match by any table pattern) are moved to this folder.",
          "type": "string"
        },
        "archive": {
          "description": "Files that have been ingested are moved to this folder if SL_ARCHIVE is set to true.",
          "type": "string"
        },
        "ingesting": {
          "description": "Files that are being ingested are moved to this folder.",
          "type": "string"
        },
        "accepted": {
          "description": "When filesystem storage is used, successfully ingested stored in this this folder in parquet format or any format set by the SL_DEFAULT_WRITE_FORMAT env property.",
          "type": "string"
        },
        "rejected": {
          "description": "When filesystem storage is used, rejected records are stored in this folder in parquet format or any format set by the SL_DEFAULT_WRITE_FORMAT env property.",
          "type": "string"
        },
        "replay": {
          "description": "Invalid records are stored in this folder in source format when SL_SINK_REPLAY_TO_FILE is set to true.",
          "type": "string"
        },
        "business": {
          "description": "",
          "type": "string"
        },
        "hiveDatabase": {
          "description": "",
          "type": "string"
        }
      }
    },
    "Freshness": {
      "type": "object",
      "properties": {
        "warn": {
          "type": "string",
          "description": "How old may be the data before a warning is raised. Use syntax like '3 day' or '2 hour' or '30 minute'"
        },
        "error": {
          "type": "string",
          "description": "How old may be the data before an error is raised. Use syntax like '3 day' or '2 hour' or '30 minute'"
        }
      }
    },
    "Table": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string",
          "description": "Schema name, must be unique among all the schemas belonging to the same domain.\n  *                     Will become the hive table name On Premise or BigQuery Table name on GCP."
        },
        "rename": {
          "type": "string",
          "description": "If present, the table is renamed with this name. Useful when use in conjunction with the 'extract' module"
        },
        "pattern": {
          "description": "filename pattern to which this schema must be applied.\n  *                     This instructs the framework to use this schema to parse any file with a filename that match this pattern.",
          "type": "string"
        },
        "attributes": {
          "description": "Attributes parsing rules.",
          "type": "array",
          "items": {
            "$ref": "#/definitions/Attribute"
          }
        },
        "metadata": {
          "$ref": "#/definitions/Metadata",
          "description": "Dataset metadata"
        },
        "merge": {
          "$ref": "#/definitions/MergeOptions"
        },
        "comment": {
          "type": "string",
          "description": "free text"
        },
        "presql": {
          "type": "array",
          "description": "Reserved for future use.",
          "items": {
            "type": "string"
          }
        },
        "postsql": {
          "type": "array",
          "description": "Reserved for future use.",
          "items": {
            "type": "string"
          }
        },
        "tags": {
          "description": "Set of string to attach to this Schema",
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "rls": {
          "description": " Row level security on this schema.",
          "type": "array",
          "items": {
            "$ref": "#/definitions/RowLevelSecurity"
          }
        },
        "expectations": {
          "$ref": "#/definitions/MapString",
          "description": "Expectations to check after Load / Transform has succeeded"
        },
        "primaryKey": {
          "description": "List of columns that make up the primary key",
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "acl": {
          "description": "Map of rolename -> List[Users].",
          "type": "array",
          "items": {
            "$ref": "#/definitions/AccessControlEntry"
          }
        },
        "sample": {
          "description": "Store here a couple of records illustrating the table data.",
          "type": "string"
        },
        "filter": {
          "description": "remove all records that do not match this condition",
          "type": "string"
        },
        "patternSample": {
          "description": "Sample of filename matching this schema",
          "type": "string"
        }
      },
      "required": ["name", "pattern"]
    },
    "Attribute": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string",
          "description": "Attribute name as defined in the source dataset and as received in the file"
        },
        "type": {
          "type": "string",
          "description": "semantic type of the attribute"
        },
        "array": {
          "type": "boolean",
          "description": "Is it an array ?"
        },
        "required": {
          "type": "boolean",
          "description": "Should this attribute always be present in the source"
        },
        "privacy": {
          "type": "string",
          "description": "Should this attribute be applied a privacy transformation at ingestion time"
        },
        "comment": {
          "type": "string",
          "description": "free text for attribute description"
        },
        "rename": {
          "type": "string",
          "description": "If present, the attribute is renamed with this name"
        },
        "metricType": {
          "type": "string",
          "description": "If present, what kind of stat should be computed for this field"
        },
        "attributes": {
          "type": "array",
          "description": "List of sub-attributes (valid for JSON and XML files only)",
          "items": {
            "$ref": "#/definitions/Attribute"
          }
        },
        "position": {
          "$ref": "#/definitions/Position"
        },
        "default": {
          "type": "string",
          "description": "Default value for this attribute when it is not present."
        },
        "tags": {
          "type": "array",
          "description": "Tags associated with this attribute",
          "items": {
            "type": "string"
          }
        },
        "trim": {
          "$ref": "#/definitions/Trim"
        },
        "script": {
          "type": "string",
          "description": "Scripted field : SQL request on renamed column"
        },
        "foreignKey": {
          "type": "string",
          "description": "If this attribute is a foreign key, reference to [domain.]table[.attribute]"
        },
        "ignore": {
          "type": "boolean",
          "description": "Should this attribute be ignored on ingestion. Default to false"
        },
        "accessPolicy": {
          "type": "string",
          "description": "Policy tag to assign to this attribute. Used for column level security"
        }
      },
      "required": ["name", "type"]
    },
    "AutoTaskDesc": {
      "type": "object",
      "properties": {
        "sql": {
          "type": "string",
          "description": "Main SQL request to execute (do not forget to prefix table names with the database name to avoid conflicts)"
        },
        "database": {
          "type": "string",
          "description": "Output Database (refer to a project id in BigQuery). Default to SL_DATABASE env var if set."
        },
        "domain": {
          "type": "string",
          "description": "Output domain in output Area (Will be the Database name in Hive or Dataset in BigQuery)"
        },
        "table": {
          "type": "string",
          "description": "Dataset Name in output Area (Will be the Table name in Hive & BigQuery)"
        },
        "write": {
          "$ref": "#/definitions/WriteMode"
        },
        "partition": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "List of columns used for partitioning the output."
        },
        "presql": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "List of SQL requests to executed before the main SQL request is run"
        },
        "postsql": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "List of SQL requests to executed after the main SQL request is run"
        },
        "sink": {
          "$ref": "#/definitions/Sink"
        },
        "rls": {
          "type": "array",
          "items": {
            "$ref": "#/definitions/RowLevelSecurity"
          }
        },
        "expectations": {
          "$ref": "#/definitions/MapString",
          "description": "Expectations to check after Load / Transform has succeeded"
        },
        "acl": {
          "description": "Map of rolename -> List[Users].",
          "type": "array",
          "items": {
            "$ref": "#/definitions/AccessControlEntry"
          }
        },
        "comment": {
          "type": "string",
          "description": "Output table description"
        },
        "freshness": {
          "description": "Configure freshness checks on the output table",
          "$ref": "#/definitions/Freshness"
        },
        "attributesDesc": {
          "description": "Attributes comments and access policies",
          "type": "array",
          "items": {
            "$ref": "#/definitions/AttributeDesc"
          }
        },
        "python": {
          "type": "string",
          "description": "Python script URI to execute instead of the SQL request"
        },
        "tags": {
          "description": "Set of string to attach to the output table",
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "merge": {
          "$ref": "#/definitions/MergeOptions"
        },
        "schedule": {
          "type": "string",
          "description": "Cron expression to use for this task"
        },
        "dagRef": {
          "type": "string",
          "description": "Cron expression to use for this domain/table"
        }
      },
      "required": []
    },
    "Lock": {
      "type": "object",
      "properties": {
        "path": {
          "type": "string",
          "description": "Name of the lock"
        },
        "timeout": {
          "type": "integer",
          "description": "Name of the lock"
        },
        "pollTime": {
          "type": "string",
          "description": "Name of the lock"
        },
        "refreshTime": {
          "type": "string",
          "description": "Name of the lock"
        }
      }
    },
    "Audit": {
      "type": "object",
      "properties": {
        "path": {
          "type": "string",
          "description": "Main SQL request to execute (do not forget to prefix table names with the database name to avoid conflicts)"
        },
        "sink": {
          "$ref": "#/definitions/Sink",
          "description": "Output Database (refer to a project id in BigQuery). Default to SL_DATABASE env var if set."
        },
        "maxErrors": {
          "type": "string",
          "description": "Output domain in output Area (Will be the Database name in Hive or Dataset in BigQuery)"
        },
        "database": {
          "type": "string",
          "description": "Dataset Name in output Area (Will be the Table name in Hive & BigQuery)"
        },
        "domain": {
          "type": "string"
        },
        "active": {
          "type": "boolean",
          "description": "Output table description"
        }
      },
      "required": []
    },
    "AttributeDesc": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string",
          "description": "Column name"
        },
        "comment": {
          "type": "string",
          "description": "Column description"
        },
        "accessPolicy": {
          "type": "string",
          "description": "Access policy to apply to this column"
        }
      },
      "required": ["name", "comment"]
    },
    "Domain": {
      "type": "object",
      "description": "A schema in JDBC database or a folder in HDFS or a dataset in BigQuery.",
      "properties": {
        "name": {
          "type": "string",
          "description": "Domain name. Make sure you use a name that may be used as a folder name on the target storage.\n                   - When using HDFS or Cloud Storage,  files once ingested are stored in a sub-directory named after the domain name.\n                   - When used with BigQuery, files are ingested and sorted in tables under a dataset named after the domain name."
        },
        "rename": {
          "type": "string",
          "description": "If present, the attribute is renamed with this name"
        },
        "metadata": {
          "$ref": "#/definitions/Metadata"
        },
        "tables": {
          "type": "array",
          "description": "List of schemas for each dataset in this domain.\nA domain usually contains multiple schemas. Each schema defining how the contents of the input file should be parsed.\nSee Schema for more details.",
          "items": {
            "$ref": "#/definitions/Table"
          }
        },
        "comment": {
          "description": "Domain Description (free text)",
          "type": "string"
        },
        "database": {
          "description": "Output Database (refer to a project id in BigQuery). Default to SL_DATABASE env var if set.",
          "type": "string"
        },
        "tags": {
          "description": "Set of string to attach to this domain",
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      }
    },
    "AutoJobDesc": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string",
          "description": "Optional name. If not specified, the name of the file without the extension is used."
        },
        "comment": {
          "type": "string",
          "description": "Optional description."
        },
        "default": {
          "$ref": "#/definitions/AutoTaskDesc",
          "description": "Default task properties to apply to all tasks defined in tasks section and in included files"
        },
        "tasks": {
          "type": "array",
          "items": {
            "$ref": "#/definitions/AutoTaskDesc",
            "description": "List of transform tasks to execute"
          }
        }
      }
    },
    "JDBCTable": {
      "type": "object",
      "properties": {
        "name": {
          "type": "string",
          "description": "table name. Set to '*' to extract all tables"
        },
        "columns": {
          "description": "List of columns to extract. All columns by default.",
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "partitionColumn": {
          "type": "string"
        },
        "numPartitions": {
          "type": "integer"
        },
        "connectionOptions": {
          "type": "string"
        },
        "fetchSize": {
          "type": "integer"
        },
        "fullExport": {
          "type": "boolean"
        }
      },
      "required": ["name"]
    },
    "JDBCSchema": {
      "type": "object",
      "properties": {
        "catalog": {
          "type": "string",
          "description": "Optional catalog name in the source database"
        },
        "schema": {
          "type": "string",
          "description": "Database schema where source tables are located"
        },
        "write": {
          "$ref": "#/definitions/WriteMode"
        },
        "pattern": {
          "type": "string"
        },
        "numericTrim": {
          "$ref": "#/definitions/Trim"
        },
        "partitionColumn": {
          "type": "string"
        },
        "numPartitions": {
          "type": "integer"
        },
        "connectionOptions": {
          "type": "string"
        },
        "fetchSize": {
          "type": "integer"
        },
        "stringPartitionFunc": {
          "type": "string"
        },
        "fullExport": {
          "type": "boolean"
        },
        "tableRemarks": {
          "type": "string"
        },
        "columnRemarks": {
          "type": "string"
        },
        "template": {
          "type": "string",
          "description": "Metadata to use for the generated YAML file."
        },
        "tables": {
          "description": "List of tables to extract",
          "type": "array",
          "items": {
            "$ref": "#/definitions/JDBCTable"
          }
        },
        "tableTypes": {
          "description": "One or many of the predefined table types",
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      }
    },
    "JDBCSchemas": {
      "type": "object",
      "properties": {
        "jdbcSchemas": {
          "description": "List database connections to use to extract the data",
          "type": "array",
          "items": {
            "$ref": "#/definitions/JDBCSchema"
          }
        },
        "default": {
          "$ref": "#/definitions/JDBCSchema"
        },
        "fetchSize": {
          "type": "integer"
        },
        "connectionRef": {
          "type": "string",
          "description": "Connection name as defined in the connections section of the application.conf file"
        },
        "connection": {
          "$ref": "#/definitions/MapString",
          "description": "JDBC connection options: url, user, password ..."
        }
      }
    },
    "RefInput": {
      "description": "Input for ref object",
      "type": "object",
      "properties": {
        "database": {
          "type": "string",
          "description": "Database pattern to match, none if any database"
        },
        "domain": {
          "type": "string",
          "description": "Domain pattern to match, none if any domain match"
        },
        "table": {
          "type": "string",
          "description": "Table pattern to match"
        }
      },
      "required": ["table"]
    },
    "RefOutput": {
      "description": "Output for ref object",
      "type": "object",
      "properties": {
        "database": {
          "type": "string",
          "description": ""
        },
        "domain": {
          "type": "string",
          "description": ""
        },
        "table": {
          "type": "string",
          "description": ""
        }
      },
      "required": ["table", "domain", "database"]
    },
    "Ref": {
      "description": "Describe how to resolve a reference in a transform task",
      "type": "object",
      "properties": {
        "input": {
          "$ref": "#/definitions/RefInput",
          "description": "The input table to resolve"
        },
        "output": {
          "$ref": "#/definitions/RefOutput",
          "description": "The output table resolved with the domain and database"
        }
      },
      "required": ["input", "output"]
    },
    "Refs": {
      "type": "array",
      "items": {
        "$ref": "#/definitions/Ref"
      }
    },
    "Application": {
      "type": "object",
      "properties": {
        "env": {
          "type": "string",
          "description": "Default environment to use. May be also set using the SL_ENV environment variable"
        },
        "datasets": {
          "type": "string",
          "description": "When using filesystem storage, default path to store the datasets"
        },
        "dags": {
          "type": "string",
          "description": "DAG generation config folder. metadata/dags by default"
        },
        "metadata": {
          "type": "string",
          "description": "default metadata folder name. May be also set using the SL_METADATA environment variable"
        },
        "metrics": {
          "$ref": "#/definitions/Metrics"
        },
        "validateOnLoad": {
          "type": "boolean",
          "description": "Validate the YAML file when loading it. If set to true fails on any error"
        },
        "audit": {
          "$ref": "#/definitions/Audit"
        },
        "archive": {
          "type": "boolean",
          "description": "Should ingested files be archived after ingestion ?"
        },
        "sinkReplayToFile": {
          "type": "boolean",
          "description": "Should invalid records be stored in a replay file ?"
        },
        "lock": {
          "$ref": "#/definitions/Lock"
        },
        "defaultFormat": {
          "type": "string",
          "description": "Default write format in Spark. parquet is the default"
        },
        "defaultRejectedWriteFormat": {
          "type": "string",
          "description": "Default write format in Spark for rejected records. parquet is the default"
        },
        "defaultAuditWriteFormat": {
          "type": "string",
          "description": "Default write format in Spark for audit records. parquet is the default"
        },
        "csvOutput": {
          "type": "boolean",
          "description": "output files in CSV format ? Default is false"
        },
        "csvOutputExt": {
          "type": "string",
          "description": "CSV file extension when csvOutput is true. Default is .csv"
        },
        "privacyOnly": {
          "type": "boolean",
          "description": "Only generate privacy tasks. Reserved for internal use"
        },
        "emptyIsNull": {
          "type": "boolean",
          "description": "Should empty strings be considered as null values ?"
        },
        "loader": {
          "type": "string",
          "description": "Default loader to use when none is specified in the schema. Valid values are 'spark' or 'native'. Default is 'spark'"
        },
        "rowValidatorClass": {
          "type": "string",
          "description": ""
        },
        "treeValidatorClass": {
          "type": "string",
          "description": ""
        },
        "loadStrategyClass": {
          "type": "string",
          "description": "In what order should the files for a same table be loaded ? By time (default) or by or name ?\n",
          "enum": [
            "ai.starlake.job.load.IngestionNameStrategy",
            "ai.starlake.job.load.IngestionTimeStrategy"
          ]
        },
        "analyze": {
          "type": "boolean",
          "description": "Should we analyze the result and generate HIVE statistics ? (useful for Spark / Databricks) "
        },
        "hive": {
          "type": "boolean",
          "description": "Should we create the table in Hive ? (useful for Spark / Databricks) "
        },
        "grouped": {
          "type": "boolean",
          "description": "Should we load of the files to be stored in the same table in a single task or one by one ?"
        },
        "groupedMax": {
          "type": "integer",
          "description": "Maximum number of files to be stored in the same table in a single task"
        },
        "mergeForceDistinct": {
          "type": "boolean",
          "description": "Should we force a distinct on the merge ?"
        },
        "mergeOptimizePartitionWrite": {
          "type": "boolean",
          "description": "Should we optimize the partition write on the merge ?"
        },
        "area": {
          "$ref": "#/definitions/Area",
          "description": "pending, accepted ... areas configuration"
        },
        "hadoop": {
          "$ref": "#/definitions/MapString",
          "description": "Hadoop configuration if applicable"
        },
        "connections": {
          "$ref": "#/definitions/MapConnection",
          "description": "Connection configurations"
        },
        "jdbcEngines": {
          "$ref": "#/definitions/MapJdbcEngine",
          "description": "JDBC engine configurations"
        },
        "privacy": {
          "$ref": "#/definitions/Privacy",
          "description": "Privacy algorithms"
        },
        "root": {
          "type": "string",
          "description": "Root folder for the application. May be also set using the SL_ROOT environment variable"
        },
        "internal": {
          "$ref": "#/definitions/Internal",
          "description": "Internal configuration"
        },
        "accessPolicies": {
          "$ref": "#/definitions/AccessPolicies",
          "description": "Access policies configuration"
        },
        "scheduling": {
          "$ref": "#/definitions/JobScheduling",
          "description": "Spark Job scheduling configuration"
        },
        "udfs": {
          "type": "string",
          "description": "Coma separated list of UDF to register in Spark jobs. May be also set using the SL_UDFS environment variable"
        },
        "expectations": {
          "$ref": "#/definitions/ExpectationsConfig",
          "description": "Expectations configuration"
        },
        "sqlParameterPattern": {
          "type": "string",
          "description": "Pattern to use to replace parameters in SQL queries in addition to the jinja syntax {{param}}. Default is ${param}"
        },
        "rejectAllOnError": {
          "type": "string",
          "description": "Should we reject all records when an error occurs ? Default is false"
        },
        "rejectMaxRecords": {
          "type": "integer",
          "description": "Maximum number of records to reject when an error occurs. Default is 100"
        },
        "maxParCopy": {
          "type": "integer",
          "description": ""
        },
        "dsvOptions": {
          "$ref": "#/definitions/MapString",
          "description": "DSV ingestion extra options"
        },
        "forceViewPattern": {
          "type": "string",
          "description": "reserved"
        },
        "forceDomainPattern": {
          "type": "string",
          "description": "reserved"
        },
        "forceTablePattern": {
          "type": "string",
          "description": "reserved"
        },
        "forceJobPattern": {
          "type": "string",
          "description": "reserved"
        },
        "forceTaskPattern": {
          "type": "string",
          "description": "reserved"
        },
        "useLocalFileSystem": {
          "type": "string",
          "description": "reserved"
        },
        "sessionDurationServe": {
          "type": "integer",
          "description": "reserved"
        },
        "database": {
          "type": "string",
          "description": "Default target database (projectId in GCP). May be also set using the SL_DATABASE environment variable"
        },
        "tenant": {
          "type": "string",
          "description": "reserved"
        },
        "connectionRef": {
          "type": "string",
          "description": "Default connection to use when loading / transforming data"
        },
        "schedulePresets": {
          "$ref": "#/definitions/MapString",
          "description": "Default connection to use when loading / transforming data"
        },
        "maxParTask": {
          "type": "integer",
          "description": "How many job to run simultaneously in dev mode (experimental)"
        }
      }
    }
  },
  "description": "JSON Schema for Starlake Data Pipeline",
  "oneOf": [
    {
      "required": ["extract"]
    },
    {
      "required": ["load"]
    },
    {
      "required": ["transform"]
    },
    {
      "required": ["expectations"]
    },
    {
      "required": ["env"]
    },
    {
      "required": ["types"]
    },
    {
      "required": ["tables"]
    },
    {
      "required": ["table"]
    },
    {
      "required": ["task"]
    },
    {
      "required": ["connections"]
    },
    {
      "required": ["external"]
    },
    {
      "required": ["application"]
    },
    {
      "required": ["refs"]
    },
    {
      "required": ["dag"]
    }
  ],
  "properties": {
    "types": {
      "type": "array",
      "items": {
        "$ref": "#/definitions/Type"
      }
    },
    "dag": {
      "$ref": "#/definitions/DagGenerationConfig"
    },
    "extract": {
      "$ref": "#/definitions/JDBCSchemas"
    },
    "load": {
      "$ref": "#/definitions/Domain"
    },
    "transform": {
      "$ref": "#/definitions/AutoJobDesc"
    },
    "task": {
      "$ref": "#/definitions/AutoTaskDesc"
    },
    "env": {
      "$ref": "#/definitions/MapString"
    },
    "expectations": {
      "$ref": "#/definitions/MapExpectationItem",
      "description": "Expectations library defined as a map name(params) -> sql request that should return 0 record"
    },
    "table": {
      "$ref": "#/definitions/Table"
    },
    "external": {
      "$ref": "#/definitions/External"
    },
    "refs": {
      "$ref": "#/definitions/Refs"
    },
    "application": {
      "$ref": "#/definitions/Application"
    }
  },
  "title": "Starlake Data Pipeline",
  "type": "object"
}
