Join the community today
Become a Member

Nginx How To Configure Nginx For JSON Based Access Logging

Discussion in 'Centmin Mod Insights' started by eva2000, May 3, 2020.

Thread Status:
Not open for further replies.
  1. eva2000

    eva2000 Administrator Staff Member

    45,728
    10,382
    113
    May 24, 2014
    Brisbane, Australia
    Ratings:
    +16,112
    Local Time:
    12:18 AM
    Nginx 1.19.x
    MariaDB 5.5/10.x
    Nginx web server supports various access log formats as outlined at Module ngx_http_log_module. Centmin Mod 123.09beta01's Nginx server has a few additional Nginx access log formats already added in /usr/local/nginx/conf/nginx.conf.

    Within /usr/local/nginx/conf/nginx.conf, they include:
    Code (Text):
    log_format  main  '$remote_addr - $remote_user [$time_local] $request '
                   '"$status" $body_bytes_sent "$http_referer" '
                   '"$http_user_agent" "$http_x_forwarded_for" "$gzip_ratio"'
                   ' "$connection" "$connection_requests" "$request_time"';
    
    log_format  ddos-proxy '$remote_addr for $http_x_real_ip - $remote_user [$time_local] $request '
                   '"$status" $body_bytes_sent "$http_referer" '
                   '"$http_user_agent" "$http_x_forwarded_for" "$gzip_ratio"'
                   ' "$connection" "$connection_requests" "$request_time"';
    
    log_format  main_ext  '$remote_addr - $remote_user [$time_local] "$request" '
                         '$status $body_bytes_sent "$http_referer" '
                         '"$http_user_agent" "$http_x_forwarded_for" '
                         '"$host" sn="$server_name" '
                         'rt=$request_time '
                         'ua="$upstream_addr" us="$upstream_status" '
                         'ut="$upstream_response_time" ul="$upstream_response_length" '
                         'cs=$upstream_cache_status' ;
    

    However, you can also setup additional Nginx access logging use JSON format as well via JSON escaping support added to log_format (escape=json) which was introduced in Nginx 1.11.8. This will increase access log disk space usage, as you'd be logging in both default format and JSON format too. So ensure you have enough disk space. Centmin Mod Nginx's logrotate will also automatically log rotate the Nginx JSON formatted access logs.

    How To Setup Nginx JSON Access Logging



    To setup Nginx JSON access logging, you will need to manually edit your main /usr/local/nginx/conf/nginx.conf config file and add a 4th log_format to existing listing which is listed directly below where log_format named = main_json:
    Code (Text):
    log_format main_json escape=json '{'
      '"msec": "$msec", ' # request unixtime in seconds with a milliseconds resolution
      '"connection": "$connection", ' # connection serial number
      '"connection_requests": "$connection_requests", ' # number of requests made in connection
      '"pid": "$pid", ' # process pid
      '"request_id": "$request_id", ' # the unique request id
      '"request_length": "$request_length", ' # request length (including headers and body)
      '"remote_addr": "$remote_addr", ' # client IP
      '"remote_user": "$remote_user", ' # client HTTP username
      '"remote_port": "$remote_port", ' # client port
      '"time_local": "$time_local", '
      '"time_iso8601": "$time_iso8601", ' # local time in the ISO 8601 standard format
      '"request": "$request", ' # full path no arguments if the request
      '"request_uri": "$request_uri", ' # full path and arguments if the request
      '"args": "$args", ' # args
      '"status": "$status", ' # response status code
      '"body_bytes_sent": "$body_bytes_sent", ' # the number of body bytes exclude headers sent to a client
      '"bytes_sent": "$bytes_sent", ' # the number of bytes sent to a client
      '"http_referer": "$http_referer", ' # HTTP referer
      '"http_user_agent": "$http_user_agent", ' # user agent
      '"http_x_forwarded_for": "$http_x_forwarded_for", ' # http_x_forwarded_for
      '"http_host": "$http_host", ' # the request Host: header
      '"server_name": "$server_name", ' # the name of the vhost serving the request
      '"request_time": "$request_time", ' # request processing time in seconds with msec resolution
      '"upstream": "$upstream_addr", ' # upstream backend server for proxied requests
      '"upstream_connect_time": "$upstream_connect_time", ' # upstream handshake time incl. TLS
      '"upstream_header_time": "$upstream_header_time", ' # time spent receiving upstream headers
      '"upstream_response_time": "$upstream_response_time", ' # time spend receiving upstream body
      '"upstream_response_length": "$upstream_response_length", ' # upstream response length
      '"upstream_cache_status": "$upstream_cache_status", ' # cache HIT/MISS where applicable
      '"ssl_protocol": "$ssl_protocol", ' # TLS protocol
      '"ssl_cipher": "$ssl_cipher", ' # TLS cipher
      '"scheme": "$scheme", ' # http or https
      '"request_method": "$request_method", ' # request method
      '"server_protocol": "$server_protocol", ' # request protocol, like HTTP/1.1 or HTTP/2.0
      '"pipe": "$pipe", ' # “p” if request was pipelined, “.” otherwise
      '"gzip_ratio": "$gzip_ratio", '
      '"http_cf_ray": "$http_cf_ray"'
    '}';

    So your /usr/local/nginx/conf/nginx.conf config file will now look like (scrolling required to view all):
    Code (Text):
    log_format  main  '$remote_addr - $remote_user [$time_local] $request '
                   '"$status" $body_bytes_sent "$http_referer" '
                   '"$http_user_agent" "$http_x_forwarded_for" "$gzip_ratio"'
                   ' "$connection" "$connection_requests" "$request_time"';
    
    log_format  ddos-proxy '$remote_addr for $http_x_real_ip - $remote_user [$time_local] $request '
                   '"$status" $body_bytes_sent "$http_referer" '
                   '"$http_user_agent" "$http_x_forwarded_for" "$gzip_ratio"'
                   ' "$connection" "$connection_requests" "$request_time"';
    
    log_format  main_ext  '$remote_addr - $remote_user [$time_local] "$request" '
                         '$status $body_bytes_sent "$http_referer" '
                         '"$http_user_agent" "$http_x_forwarded_for" '
                         '"$host" sn="$server_name" '
                         'rt=$request_time '
                         'ua="$upstream_addr" us="$upstream_status" '
                         'ut="$upstream_response_time" ul="$upstream_response_length" '
                         'cs=$upstream_cache_status' ;
    
    log_format main_json escape=json '{'
      '"msec": "$msec", ' # request unixtime in seconds with a milliseconds resolution
      '"connection": "$connection", ' # connection serial number
      '"connection_requests": "$connection_requests", ' # number of requests made in connection
      '"pid": "$pid", ' # process pid
      '"request_id": "$request_id", ' # the unique request id
      '"request_length": "$request_length", ' # request length (including headers and body)
      '"remote_addr": "$remote_addr", ' # client IP
      '"remote_user": "$remote_user", ' # client HTTP username
      '"remote_port": "$remote_port", ' # client port
      '"time_local": "$time_local", '
      '"time_iso8601": "$time_iso8601", ' # local time in the ISO 8601 standard format
      '"request": "$request", ' # full path no arguments if the request
      '"request_uri": "$request_uri", ' # full path and arguments if the request
      '"args": "$args", ' # args
      '"status": "$status", ' # response status code
      '"body_bytes_sent": "$body_bytes_sent", ' # the number of body bytes exclude headers sent to a client
      '"bytes_sent": "$bytes_sent", ' # the number of bytes sent to a client
      '"http_referer": "$http_referer", ' # HTTP referer
      '"http_user_agent": "$http_user_agent", ' # user agent
      '"http_x_forwarded_for": "$http_x_forwarded_for", ' # http_x_forwarded_for
      '"http_host": "$http_host", ' # the request Host: header
      '"server_name": "$server_name", ' # the name of the vhost serving the request
      '"request_time": "$request_time", ' # request processing time in seconds with msec resolution
      '"upstream": "$upstream_addr", ' # upstream backend server for proxied requests
      '"upstream_connect_time": "$upstream_connect_time", ' # upstream handshake time incl. TLS
      '"upstream_header_time": "$upstream_header_time", ' # time spent receiving upstream headers
      '"upstream_response_time": "$upstream_response_time", ' # time spend receiving upstream body
      '"upstream_response_length": "$upstream_response_length", ' # upstream response length
      '"upstream_cache_status": "$upstream_cache_status", ' # cache HIT/MISS where applicable
      '"ssl_protocol": "$ssl_protocol", ' # TLS protocol
      '"ssl_cipher": "$ssl_cipher", ' # TLS cipher
      '"scheme": "$scheme", ' # http or https
      '"request_method": "$request_method", ' # request method
      '"server_protocol": "$server_protocol", ' # request protocol, like HTTP/1.1 or HTTP/2.0
      '"pipe": "$pipe", ' # “p” if request was pipelined, “.” otherwise
      '"gzip_ratio": "$gzip_ratio", '
      '"http_cf_ray": "$http_cf_ray"'
    '}';

    Next in your Nginx site vhost config file i.e. HTTPS vhost would be at /usr/local/nginx/conf/conf.d/domain.com.ssl.conf, you will need to add a new access_log line referencing the log_format named = main_json. The buffer and flush directives tell Nginx to use a memory buffer to write to access log every 256KB size or after 5 minutes for better performance. So you will need to do Nginx reload/restart to flush any memory buffered access logs to disk if you want to inspect latest log info.
    Code (Text):
    access_log /home/nginx/domains/domain.com/log/access_log.json main_json buffer=256k flush=5m;
    

    Normally, you would want to place it grouped with your existing access_log in your Nginx site vhost config file like.
    Code (Text):
    access_log /home/nginx/domains/domain.com/log/access.log combined buffer=256k flush=5m;
    access_log /home/nginx/domains/domain.com/log/access_log.json main_json buffer=256k flush=5m;
    error_log /home/nginx/domains/domain.com/log/error.log;
    

    Then restart Nginx server for changes to take affect
    Code (Text):
    service nginx restart

    or cmd shortcut
    Code (Text):
    ngxrestart

    Centmin Mod 123.09beta01 and newer installs already add jq command line to allow to your parse and prettify your JSON output.

    Example output for last line (tail -1) of access_log.json log file using commands which pipe the JSON log output through jq command:
    Code (Text):
    cd /home/nginx/domains/domain.com/log/
    cat access_log.json | tail -1 | jq -r .
    

    Code (Text):
    cd /home/nginx/domains/domain.com/log/
    
    cat access_log.json | tail -1 | jq -r .
    {
      "http_cf_ray": "58d8c8f393bbe049-DFW",
      "gzip_ratio": "4.50",
      "pipe": ".",
      "server_protocol": "HTTP/1.1",
      "request_method": "GET",
      "body_bytes_sent": "23142",
      "status": "200",
      "args": "/threads/ovh-new-infrastructure-line-severs-intel-xeon-e-22xx-xeon-silver-amd-epyc-7371.18392/&",
      "request_uri": "/threads/ovh-new-infrastructure-line-severs-intel-xeon-e-22xx-xeon-silver-amd-epyc-7371.18392/",
      "request": "GET /threads/ovh-new-infrastructure-line-severs-intel-xeon-e-22xx-xeon-silver-amd-epyc-7371.18392/ HTTP/1.1",
      "time_iso8601": "2020-05-03T08:59:16+00:00",
      "time_local": "03/May/2020:08:59:16 +0000",
      "remote_port": "",
      "msec": "1588496356.551",
      "connection": "5908",
      "connection_requests": "1",
      "pid": "18672",
      "request_id": "29e7f4290c9ecf06e780552b25883aba",
      "request_length": "875",
      "remote_addr": "66.249.73.203",
      "remote_user": "",
      "bytes_sent": "23992",
      "http_referer": "",
      "http_user_agent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
      "http_x_forwarded_for": "66.249.73.203",
      "http_host": "community.centminmod.com",
      "server_name": "community.centminmod.com",
      "request_time": "0.100",
      "upstream": "127.0.0.1:9000",
      "upstream_connect_time": "0.000",
      "upstream_header_time": "0.100",
      "upstream_response_time": "0.100",
      "upstream_response_length": "104181",
      "upstream_cache_status": "BYPASS",
      "ssl_protocol": "TLSv1.3",
      "ssl_cipher": "TLS_AES_256_GCM_SHA384",
      "scheme": "https"
    }
    

    You can use jq to filter just the Nginx metrics you want. Example
    Code (Text):
    cat access_log.json | tail -1 | jq -r '"\(.time_local) \(.remote_addr) \(.http_x_forwarded_for) \(.request_uri) \(.status) \(.request_method) \(.http_host) \(.scheme) \(.server_protocol) \(.ssl_protocol) \(.ssl_cipher) \(.http_cf_ray) \(.http_user_agent)"'
    03/May/2020:08:59:16 +0000 66.249.73.203 66.249.73.203 /threads/ovh-new-infrastructure-line-severs-intel-xeon-e-22xx-xeon-silver-amd-epyc-7371.18392/ 200 GET community.centminmod.com https HTTP/1.1 TLSv1.3 TLS_AES_256_GCM_SHA384 58d8c8f393bbe049-DFW Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
    

    Or find out the proportion of HTTPS SSL ciphers served to visitors
    Code (Text):
    cat access_log.json | jq -r '.ssl_cipher' | sort | uniq -c | sort -rn
       3099 TLS_AES_256_GCM_SHA384
        171 ECDHE-ECDSA-AES256-GCM-SHA384
         22 ECDHE-ECDSA-CHACHA20-POLY1305
          2 ECDHE-ECDSA-AES128-SHA
    

    HTTPS SSL protocols and ciphers served to visitors
    Code (Text):
    cat access_log.json | jq -r '"\(.ssl_protocol) \(.ssl_cipher)"' | sort | uniq -c | sort -rn
       3099 TLSv1.3 TLS_AES_256_GCM_SHA384
        171 TLSv1.2 ECDHE-ECDSA-AES256-GCM-SHA384
         22 TLSv1.2 ECDHE-ECDSA-CHACHA20-POLY1305
          1 TLSv1 ECDHE-ECDSA-AES128-SHA
          1 TLSv1.1 ECDHE-ECDSA-AES128-SHA
    

    HTTPS SSL protocols and HTTP status codes served to visitors
    Code (Text):
    cat access_log.json | jq -r '"\(.ssl_protocol) \(.status)"' | sort | uniq -c | sort -rn
       2115 TLSv1.3 200
        478 TLSv1.3 301
        226 TLSv1.3 307
        150 TLSv1.2 200
        135 TLSv1.3 304
         76 TLSv1.3 303
         62 TLSv1.3 403
         25 TLSv1.2 304
          6 TLSv1.2 301
          5 TLSv1.3 404
          4 TLSv1.2 307
          3 TLSv1.2 403
          3 TLSv1.2 303
          2 TLSv1.2 400
          1 TLSv1 400
          1 TLSv1.3 405
          1 TLSv1.3 400
          1 TLSv1.1 400
    

    HTTPS protocol and HTTP status codes for only urls = /login/login
    Code (Text):
    cat access_log.json | jq -r 'select(.request_uri == "/login/login")| "\(.ssl_protocol) \(.status)"' | sort | uniq -c | sort -rn                              
         25 TLSv1.3 200
    

    Top 10 requested urls for just /forums/ and /threads/ excluding rss, manifest.json and sw.js files
    Code (Text):
    cat access_log.json | egrep '/forums/|/threads/' | egrep -v 'index.rss|manifest.json|sw.js' | jq -r '"\(.ssl_protocol) \(.status) \(.request_uri)"' | sort | uniq -c | sort -rn | head -n10
       2173 TLSv1.3 200 /threads/cloudflare-domain-registrar-transfers-reduced-my-costs-by-37.16038/
        945 TLSv1.3 200 /login/csrf-token-refresh
        115 TLSv1.3 200 /login/login
        113 TLSv1.3 200 /threads/php-7-3-vs-7-2-vs-7-1-vs-7-0-php-fpm-benchmarks.16090/
        106 TLSv1.3 200 /threads/php-benchmarks-7-4-vs-7-3-vs-7-2-vs-7-1-vs-7-0-php-fpm.18741/
        103 TLSv1.3 200 /threads/php-7-x-benchmarks-centmin-mod-vs-easyengine-vs-webinoly-vs-vestacp-vs-oneinstack.14988/
        103 TLSv1.3 200 /threads/13-way-vps-server-benchmark-comparison-tests-upcloud-vs-digitalocean-vs-linode-vs-vultr-vs-hetzner.17742/
        102 TLSv1.3 200 /threads/my-first-wordpress-autoptimize-gzip-companion-plugin.15314/
        100 TLSv1.3 200 /threads/python-2-7-will-reach-the-end-of-its-life-on-january-1st-2020-running-out-of-time.18806/
         98 TLSv1.3 200 /forums/centmin-mod-news.3/
    
     
  2. eva2000

    eva2000 Administrator Staff Member

    45,728
    10,382
    113
    May 24, 2014
    Brisbane, Australia
    Ratings:
    +16,112
    Local Time:
    12:18 AM
    Nginx 1.19.x
    MariaDB 5.5/10.x

    Nginx JSON Format Logs with Cloudflare Proxy



    If you have Cloudflare proxy in front of Nginx, you can alter the Nginx json log format added to /usr/local/nginx/conf/nginx.conf to below version for additional logged fields

    log_format named = cf_json
    Code (Text):
    log_format cf_json escape=json '{'
      '"msec": "$msec", ' # request unixtime in seconds with a milliseconds resolution
      '"connection": "$connection", ' # connection serial number
      '"connection_requests": "$connection_requests", ' # number of requests made in connection
      '"pid": "$pid", ' # process pid
      '"request_id": "$request_id", ' # the unique request id
      '"request_length": "$request_length", ' # request length (including headers and body)
      '"remote_addr": "$remote_addr", ' # client IP
      '"remote_user": "$remote_user", ' # client HTTP username
      '"remote_port": "$remote_port", ' # client port
      '"time_local": "$time_local", '
      '"time_iso8601": "$time_iso8601", ' # local time in the ISO 8601 standard format
      '"request": "$request", ' # full path no arguments if the request
      '"request_uri": "$request_uri", ' # full path and arguments if the request
      '"args": "$args", ' # args
      '"status": "$status", ' # response status code
      '"body_bytes_sent": "$body_bytes_sent", ' # the number of body bytes exclude headers sent to a client
      '"bytes_sent": "$bytes_sent", ' # the number of bytes sent to a client
      '"http_referer": "$http_referer", ' # HTTP referer
      '"http_user_agent": "$http_user_agent", ' # user agent
      '"http_x_forwarded_for": "$http_x_forwarded_for", ' # http_x_forwarded_for
      '"http_host": "$http_host", ' # the request Host: header
      '"server_name": "$server_name", ' # the name of the vhost serving the request
      '"request_time": "$request_time", ' # request processing time in seconds with msec resolution
      '"upstream": "$upstream_addr", ' # upstream backend server for proxied requests
      '"upstream_connect_time": "$upstream_connect_time", ' # upstream handshake time incl. TLS
      '"upstream_header_time": "$upstream_header_time", ' # time spent receiving upstream headers
      '"upstream_response_time": "$upstream_response_time", ' # time spend receiving upstream body
      '"upstream_response_length": "$upstream_response_length", ' # upstream response length
      '"upstream_cache_status": "$upstream_cache_status", ' # cache HIT/MISS where applicable
      '"ssl_protocol": "$ssl_protocol", ' # TLS protocol
      '"ssl_cipher": "$ssl_cipher", ' # TLS cipher
      '"scheme": "$scheme", ' # http or https
      '"request_method": "$request_method", ' # request method
      '"server_protocol": "$server_protocol", ' # request protocol, like HTTP/1.1 or HTTP/2.0
      '"pipe": "$pipe", ' # “p” if request was pipelined, “.” otherwise
      '"gzip_ratio": "$gzip_ratio", '
      '"http_cf_ray": "$http_cf_ray", '
      '"http_cf_request_id": "$http_cf_request_id", '
      '"http_cf_railgun": "$http_cf_railgun", '
      '"http_accept": "$http_accept", '
      '"http_args": "$args"'
    '}';

    and use this in nginx vhost adding a 2nd log for
    /home/nginx/domains/domain.com/log/access_log.json using nginx log format = cf_json
    Code (Text):
    access_log /home/nginx/domains/domain.com/log/access.log combined buffer=256k flush=5m;
    access_log /home/nginx/domains/domain.com/log/access_log.json cf_json buffer=256k flush=5m;
    error_log /home/nginx/domains/domain.com/log/error.log;
    
     
Thread Status:
Not open for further replies.