FME Flow

{
  "elasticsearch": {
    "url": "http://elastic.host.example.com:9200",
    "username": "elastic",
    "password": "<elastic_pwd>",
    "job_index": "ct-fme-jobs",
    "log_index": "ct-fme-log",
    "job_route_index": "ct-fme-jobroutes"
  },
  "fme": {
    "token": "<token>",
    "url": "http://fme.host.example.com",
    "pagesize": 1000,
    "stage": "test"
  },
  "pipeline": {
    "job_logs": {
      "enabled": true,
      "batch_size": 20
    }
  }
}

python -m pipelines.fmejob -c /opt/conterra/fme/config.json

#!/bin/sh
python -m pipelines.fmejob -c /opt/conterra/fme/config.json

*/10 * * * * /home/monitor/fme-python/job-run.sh

# fme.base.path: '/var/log/fme'
fme.base.path: ''

# The value of this option will be added to the "labels.env" field in the index.
fme.env: 'production'

filebeat.inputs:
  - type: "filestream"
    id: "fme_server_core"
    enabled: true
    fields_under_root: true
    fields:
      labels:
        env: ${fme.env}
        source: "fme_server_core"
        pipeline: "ct-monitor-fme-log"
    paths:
      - "${fme.base.path}/logs/core/current/*.log"
    parsers:
      - multiline:
          type: "pattern"
          pattern: '^(\w{3}-\d{2}-\w{3}-\d{4} \d{2}:\d{2}:\d{2}.\d{3})\s((AM|PM)+)'
          negate: true
          match: "after"
          skip_newline: true

[... more inputs ...]

# ======================= Elasticsearch template setting =======================

setup.ilm:
  enabled: true # setting this to true will ignore setup.template.name, setup.template.pattern and output.elasticsearch index
  policy_name: "ct-fme-log-policy"
  overwrite: false
  rollover_alias: "ct-fme-log"
  pattern: "{now/d}-0000001"

# This section is ignored, when setup.ilm.enabled is true
# setup.template.name: "ct-fme-log"
# setup.template.pattern: "ct-fme-log-*"

# ================================== Outputs ===================================

# --------------------------- Elasticsearch Output -----------------------------

output.elasticsearch:
  # the index configuration is ignored when setup.ilm.enabled is true. In this case the setup.ilm.rollover_alias is used as target.
  # index: "ct-fme-log-*"
  # The name of the ingest pipeline processing the filebeat input.
  pipeline: "ct-monitor-fme-log"
  # Elasticsearch host and port
  hosts: ["https://localhost:9200"]
  # Elasticsearch user name
  username: ""
  # Elasticsearch password
  password: ""
  ssl:
    enabled: true
    # Elasticsearch SSL fingerprint
    ca_trusted_fingerprint: ""