How to write Nginx Application Performance Management Logs in JSON format

To write Nginx Application Performance Management Logs, all you need to do is just create your own logging format. To do the same follow the below steps

1. Add your own log_format in nginx.conf

Open /etc/nginx/nginx.conf and add the following lines to the http section

log_format apm escape=json    '{ "time_local" : "$time_local", '
                              '"client" : "$remote_addr", '
                              '"method" : "$request_method", '
                              '"request" : "$request", '
                              '"request_length" : $request_length, '
                              '"status" : $status, '
                              '"bytes_sent" : $bytes_sent, '
                              '"body_bytes_sent" : $body_bytes_sent, '
                              '"referer" : "$http_referer", '
                              '"user_agent" : "$http_user_agent", '
                              '"upstream_addr" : "$upstream_addr", '
                              '"upstream_status" : $upstream_status, '
                              '"request_time" : $request_time, '
                              '"upstream_response_time" : $upstream_response_time , '
                              '"upstream_connect_time" : $upstream_connect_time , '
                              '"upstream_header_time" : $upstream_header_time }';

Note: this logs numeric/ floating point logs in the respective formats instead Strings, This is handy if you want to aggregate logs.

After Adding this your nginx.conf should look something like below

user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;

events {
        worker_connections 768;
        # multi_accept on;
}

http {

        ##
        # Basic Settings
        ##

        sendfile on;
        tcp_nopush on;
        tcp_nodelay on;
        keepalive_timeout 65;
        types_hash_max_size 2048;
        # server_tokens off;

        # server_names_hash_bucket_size 64;
        # server_name_in_redirect off;

        include /etc/nginx/mime.types;
        default_type application/octet-stream;

        ##
        # SSL Settings
        ##

        ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
        ssl_prefer_server_ciphers on;

        ##
        # Logging Settings
        ##

        access_log /var/log/nginx/access.log;
        error_log /var/log/nginx/error.log;

        log_format apm escape=json    '{ "time_local" : "$time_local", '
                              '"client" : "$remote_addr", '
                              '"method" : "$request_method", '
                              '"request" : "$request", '
                              '"request_length" : $request_length, '
                              '"status" : $status, '
                              '"bytes_sent" : $bytes_sent, '
                              '"body_bytes_sent" : $body_bytes_sent, '
                              '"referer" : "$http_referer", '
                              '"user_agent" : "$http_user_agent", '
                              '"upstream_addr" : "$upstream_addr", '
                              '"upstream_status" : $upstream_status, '
                              '"request_time" : $request_time, '
                              '"upstream_response_time" : $upstream_response_time , '
                              '"upstream_connect_time" : $upstream_connect_time , '
                              '"upstream_header_time" : $upstream_header_time }';

        ##
        # Gzip Settings
        ##

        gzip on;
        gzip_disable "msie6";

        # gzip_vary on;
        # gzip_proxied any;
        # gzip_comp_level 6;
        # gzip_buffers 16 8k;
        # gzip_http_version 1.1;
        # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;

        ##
        # Virtual Host Configs
        ##

        include /etc/nginx/conf.d/*.conf;
        include /etc/nginx/sites-enabled/*;
}


#mail {
#       # See sample authentication script at:
#       # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
#
#       # auth_http localhost/auth.php;
#       # pop3_capabilities "TOP" "USER";
#       # imap_capabilities "IMAP4rev1" "UIDPLUS";
#
#       server {
#               listen     localhost:110;
#               protocol   pop3;
#               proxy      on;
#       }
#
#       server {
#               listen     localhost:143;
#               protocol   imap;
#               proxy      on;
#       }
#}

2. Use the log_format you created in your server configurations

Goto /etc/nginx/sites-available and open your server configuration file and update the access_log format to use apm
Your log configuration will look like below

access_log /var/log/nginx/yourdomain.com apm;

After adding this your server configuration file should look like below

server {
    listen 0.0.0.0:80;
    server_name yourdomain.com;
    access_log /var/log/nginx/yourdomain.com apm;
    location / {
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header HOST $http_host;
        proxy_set_header X-NginX-Proxy true;
        proxy_pass http://127.0.0.1:8080;
        proxy_redirect off;
    }
}

3. Restart the Nginx to see the effect

Now restart the Nginx using the following command, and you should start seeing JSON logs in your log files.

sudo systemctl restart nginx