Datadog入門：監視とオブザーバビリティの違いから導入・設定方法まで徹底解説

[root@appserver ~]# DD_API_KEY=52bb3bd609f81c1ced94d58138b4f892 \
DD_SITE="ap1.datadoghq.com" \
DD_APM_INSTRUMENTATION_ENABLED=host \
DD_APM_INSTRUMENTATION_LIBRARIES=java:1,python:3,js:5,php:1,dotnet:3,ruby:2 \
bash -c "$(curl -L https://install.datadoghq.com/scripts/install_script_agent7.sh)"
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 82305  100 82305    0     0   604k      0 --:--:-- --:--:-- --:--:--  604k

* Datadog Agent 7 install script v1.40.0

/usr/bin/systemctl

* Installing YUM sources for Datadog

Cache was expired
43 files removed

  Installing package(s): datadog-agent

Last metadata expiration check: 0:01:54 ago on Fri 19 Sep 2025 06:41:35 AM JST.
Dependencies resolved.
================================================================================
 Package               Architecture   Version             Repository       Size
================================================================================
Installing:
 datadog-agent         x86_64         1:7.70.2-1          datadog         154 M

Transaction Summary
================================================================================
Install  1 Package

Total download size: 154 M
Installed size: 154 M
Downloading Packages:
datadog-agent-7.70.2-1.x86_64.rpm                18 MB/s | 154 MB     00:08
--------------------------------------------------------------------------------
Total                                            18 MB/s | 154 MB     00:08
Running transaction check
Transaction check succeeded.
Running transaction test
Transaction test succeeded.
Running transaction
  Preparing        :                                                        1/1
  Running scriptlet: datadog-agent-1:7.70.2-1.x86_64                        1/1
Failed to stop datadog-agent-process.service: Unit datadog-agent-process.service not loaded.
Failed to stop datadog-agent-sysprobe.service: Unit datadog-agent-sysprobe.service not loaded.
Failed to stop datadog-agent-trace.service: Unit datadog-agent-trace.service not loaded.
Failed to stop datadog-agent-security.service: Unit datadog-agent-security.service not loaded.
Failed to stop datadog-agent.service: Unit datadog-agent.service not loaded.
Failed to disable unit: Unit file datadog-agent-process.service does not exist.
Failed to disable unit: Unit file datadog-agent-sysprobe.service does not exist.
Failed to disable unit: Unit file datadog-agent-trace.service does not exist.
Failed to disable unit: Unit file datadog-agent-security.service does not exist.
Failed to disable unit: Unit file datadog-agent.service does not exist.

  Installing       : datadog-agent-1:7.70.2-1.x86_64                        1/1
  Running scriptlet: datadog-agent-1:7.70.2-1.x86_64                        1/1
Creating file: '/opt/datadog-agent/.post_python_installed_packages.txt'
File '/opt/datadog-agent/.diff_python_installed_packages.txt' not found.

  Verifying        : datadog-agent-1:7.70.2-1.x86_64                        1/1

Installed:
  datadog-agent-1:7.70.2-1.x86_64

Complete!

* Adding your API key to the Datadog Agent configuration: /etc/datadog-agent/datadog.yaml


* Setting SITE in the Datadog Agent configuration: /etc/datadog-agent/datadog.yaml

/usr/bin/systemctl
* Starting the Datadog Agent...

  Your Datadog Agent is running and functioning properly.
  It will continue to run in the background and submit metrics to Datadog.
  If you ever want to stop the Datadog Agent, run:

       systemctl stop datadog-agent

  And to run it again run:

       systemctl start datadog-agent

[root@appserver ~]#

[root@appserver jmx.d]# systemctl status datadog-agent
 datadog-agent.service - Datadog Agent
     Loaded: loaded (/usr/lib/systemd/system/datadog-agent.service; enabled; preset: disabled)
     Active: active (running) since Sun 2025-09-21 06:32:15 JST; 5s ago
   Main PID: 3897554 (agent)
      Tasks: 9 (limit: 23168)
     Memory: 33.2M
        CPU: 4.844s
     CGroup: /system.slice/datadog-agent.service
             3897554 /opt/datadog-agent/bin/agent/agent run -p /opt/datadog-agent/run/agent.pid

Sep 21 06:32:19 appserver agent[3897554]: 2025-09-21 06:32:19 JST | CORE | INFO | (comp/forwarder/defaultforwa>
Sep 21 06:32:19 appserver agent[3897554]: 2025-09-21 06:32:19 JST | CORE | INFO | (comp/core/ipc/impl@v0.70.2/>
Sep 21 06:32:19 appserver agent[3897554]: 2025-09-21 06:32:19 JST | CORE | INFO | (comp/core/ipc/impl@v0.70.2/>
Sep 21 06:32:19 appserver agent[3897554]: 2025-09-21 06:32:19 JST | CORE | INFO | (pkg/process/metadata/worklo>
Sep 21 06:32:20 appserver agent[3897554]: 2025-09-21 06:32:20 JST | CORE | INFO | (comp/core/tagger/impl/tagge>
Sep 21 06:32:21 appserver agent[3897554]: 2025-09-21 06:32:21 JST | CORE | INFO | (comp/forwarder/defaultforwa>
Sep 21 06:32:21 appserver agent[3897554]: 2025-09-21 06:32:21 JST | CORE | INFO | (pkg/aggregator/demultiplexe>
Sep 21 06:32:21 appserver agent[3897554]: 2025-09-21 06:32:21 JST | CORE | INFO | (pkg/aggregator/demultiplexe>
Sep 21 06:32:21 appserver agent[3897554]: 2025-09-21 06:32:21 JST | CORE | INFO | (pkg/aggregator/time_sampler>
Sep 21 06:32:21 appserver agent[3897554]: 2025-09-21 06:32:21 JST | CORE | INFO | (pkg/runtime/runtime.go:28 i>
Sep 21 06:32:21 appserver agent[3897554]: 2025-09-21 06:32:21 JST | CORE | INFO | (comp/logs/agent/agentimpl/a>
lines 1-21/21 (END)

[root@appserver ~]# vi /etc/datadog-agent/conf.d/jmx.d/conf.yaml
[root@appserver ~]# cat /etc/datadog-agent/conf.d/jmx.d/conf.yaml
instances:

  - host: localhost
    port: 9010  # Tomcat JMX port
    name: tomcat
    collect_default_metrics: false

    conf:
      # Full GC (Old Generation)
      - include:
          domain: java.lang
          bean: type=GarbageCollector,name=G1\ Old\ Generation
          attribute:
            - CollectionCount
            - CollectionTime
          metric_prefix: jvm.gc.old

      # Young GC (Eden)
      - include:
          domain: java.lang
          bean: type=GarbageCollector,name=G1\ Young\ Generation
          attribute:
            - CollectionCount
            - CollectionTime
          metric_prefix: jvm.gc.new

      # Heap New (Eden space)
      - include:
          domain: java.lang
          bean: type=MemoryPool,name=G1\ Eden\ Space
          attribute:
            - Usage.used
            - Usage.max
          metric_prefix: jvm.memory.new

      # Heap Old
      - include:
          domain: java.lang
          bean: type=MemoryPool,name=G1\ Old\ Gen
          attribute:
            - Usage.used
            - Usage.max
          metric_prefix: jvm.memory.old

      # Metaspace (Java8+)
      - include:
          domain: java.lang
          bean: type=MemoryPool,name=Metaspace
          attribute:
            - Usage.used
            - Usage.max
          metric_prefix: jvm.memory.metaspace

      # Thread metrics
      - include:
          domain: java.lang
          bean: type=Threading
          attribute:
            - ThreadCount
            - PeakThreadCount
            - DaemonThreadCount
          metric_prefix: jvm.threads

      # Class loading metrics
      - include:
          domain: java.lang
          bean: type=ClassLoading
          attribute:
            - LoadedClassCount
            - UnloadedClassCount
          metric_prefix: jvm.classes

      # CPU usage metrics
      - include:
          domain: java.lang
          bean: type=OperatingSystem
          attribute:
            - ProcessCpuLoad
            - SystemCpuLoad
          metric_prefix: jvm.cpu

      # Tomcat connector threads
      - include:
          domain: Catalina
          bean: type=ThreadPool,name=http-nio-8080
          attribute:
            - currentThreadCount
            - currentThreadsBusy
            - maxThreads
          metric_prefix: tomcat.threads

      # Tomcat session metrics
      - include:
          domain: Catalina
          bean: type=Manager,context=/,host=localhost
          attribute:
            - activeSessions
            - sessionCounter
          metric_prefix: tomcat.sessions

[root@appserver ~]# systemctl restart datadog-agent
[root@appserver ~]# systemctl status datadog-agent
 datadog-agent.service - Datadog Agent
     Loaded: loaded (/usr/lib/systemd/system/datadog-agent.service; enabled; preset: disabled)
     Active: active (running) since Sun 2025-09-21 06:32:15 JST; 5s ago
   Main PID: 3897554 (agent)
      Tasks: 9 (limit: 23168)
     Memory: 33.2M
        CPU: 4.844s
     CGroup: /system.slice/datadog-agent.service
             3897554 /opt/datadog-agent/bin/agent/agent run -p /opt/datadog-agent/run/agent.pid

Sep 21 06:32:19 appserver agent[3897554]: 2025-09-21 06:32:19 JST | CORE | INFO | (comp/forwarder/defaultforwa>
Sep 21 06:32:19 appserver agent[3897554]: 2025-09-21 06:32:19 JST | CORE | INFO | (comp/core/ipc/impl@v0.70.2/>
Sep 21 06:32:19 appserver agent[3897554]: 2025-09-21 06:32:19 JST | CORE | INFO | (comp/core/ipc/impl@v0.70.2/>
Sep 21 06:32:19 appserver agent[3897554]: 2025-09-21 06:32:19 JST | CORE | INFO | (pkg/process/metadata/worklo>
Sep 21 06:32:20 appserver agent[3897554]: 2025-09-21 06:32:20 JST | CORE | INFO | (comp/core/tagger/impl/tagge>
Sep 21 06:32:21 appserver agent[3897554]: 2025-09-21 06:32:21 JST | CORE | INFO | (comp/forwarder/defaultforwa>
Sep 21 06:32:21 appserver agent[3897554]: 2025-09-21 06:32:21 JST | CORE | INFO | (pkg/aggregator/demultiplexe>
Sep 21 06:32:21 appserver agent[3897554]: 2025-09-21 06:32:21 JST | CORE | INFO | (pkg/aggregator/demultiplexe>
Sep 21 06:32:21 appserver agent[3897554]: 2025-09-21 06:32:21 JST | CORE | INFO | (pkg/aggregator/time_sampler>
Sep 21 06:32:21 appserver agent[3897554]: 2025-09-21 06:32:21 JST | CORE | INFO | (pkg/runtime/runtime.go:28 i>
Sep 21 06:32:21 appserver agent[3897554]: 2025-09-21 06:32:21 JST | CORE | INFO | (comp/logs/agent/agentimpl/a>
lines 1-21/21 (END)
[root@appserver ~]#

[root@appserver ~]# vi /etc/datadog-agent/datadog.yaml
[root@appserver ~]# cat /etc/datadog-agent/datadog.yaml
#########################
## Basic Configuration ##
#########################

## @param api_key - string - required
## @env DD_API_KEY - string - required
## The Datadog API key used by your Agent to submit metrics and events to Datadog.
## Create a new API key here: https://app.datadoghq.com/organization-settings/api-keys .
## Read more about API keys here: https://docs.datadoghq.com/account_management/api-app-keys/#api-keys .
api_key: 52bb3bd609fXXXXXXXXXXXXXXXXXXXX

~Output Truncated~

##################################
## Log collection Configuration ##
##################################

## @param logs_enabled - boolean - optional - default: false
## @env DD_LOGS_ENABLED - boolean - optional - default: false
## Enable Datadog Agent log collection by setting logs_enabled to true.
#
logs_enabled: true

~Output Truncated~

[root@appserver ~]# vi /etc/datadog-agent/conf.d/apache.d/conf.yaml
[root@appserver ~]# cat /etc/datadog-agent/conf.d/apache.d/conf.yaml
logs:
  # Apache access logs
  - type: file
    path: /var/log/httpd/access_log
    service: apache
    source: apache
    sourcecategory: http_access

  # Apache error logs
  - type: file
    path: /var/log/httpd/error_log
    service: apache
    source: apache
    sourcecategory: http_error
    # Optional: only collect WARN, ERROR, CRIT, ALERT, EMERG
    log_processing_rules:
      - type: include_at_match
        name: only_errors
        pattern: "(warn|error|crit|alert|emerg)"

[root@appserver ~]#

[root@appserver ~]# setfacl -m u:dd-agent:x /var/log/httpd
[root@appserver ~]# setfacl -m u:dd-agent:r /var/log/httpd/access_log
[root@appserver ~]# setfacl -m u:dd-agent:r /var/log/httpd/error_log
[root@appserver ~]# sudo -u dd-agent tail -n 3 /var/log/httpd/error_log

[root@appserver ~]# sudo -u dd-agent tail -n 3 /var/log/httpd/access_log
147.185.132.103 - - [22/Sep/2025:06:24:17 +0900] "\x16\x03\x01" 400 226 "-" "-"
45.135.193.100 - - [22/Sep/2025:06:27:05 +0900] "GET / HTTP/1.1" 302 - "-" "-"
91.224.92.34 - - [22/Sep/2025:06:55:10 +0900] "\x16\x03\x01\x05\xa8\x01" 400 226 "-" "-"
[root@appserver ~]#
[root@appserver ~]# sudo -u dd-agent tail -n 3 /var/log/httpd/error_log
[Sun Sep 21 05:51:18.192859 2025] [proxy_http:error] [pid 3803544:tid 3803707] [client 106.72.183.131:5181] AH01114: HTTP: failed to make connection to backend: localhost
[Sun Sep 21 05:51:18.500838 2025] [proxy:error] [pid 3806244:tid 3806274] (111)Connection refused: AH00957: http: attempt to connect to 127.0.0.1:8080 (localhost:8080) failed
[Sun Sep 21 05:51:18.501112 2025] [proxy_http:error] [pid 3806244:tid 3806274] [client 106.72.183.131:25662] AH01114: HTTP: failed to make connection to backend: localhost, referer: https://quiz.eeengineer.com/quizapp
[root@appserver ~]#

[root@appserver ~]# vi /etc/datadog-agent/datadog.yaml
[root@appserver ~]# cat /etc/datadog-agent/datadog.yaml
~Output Truncated~

####################################
## Trace Collection Configuration ##
####################################

## @param apm_config - custom object - optional
## Enter specific configurations for your trace collection.
## Uncomment this parameter and the one below to enable them.
## See https://docs.datadoghq.com/agent/apm/
#
apm_config:

  ## @param enabled - boolean - optional - default: true
  ## @env DD_APM_ENABLED - boolean - optional - default: true
  ## Set to true to enable the APM Agent.
  #
  enabled: true

  ## @param env - string - optional - default: none
  ## @env DD_APM_ENV - string - optional - default: none
  ## The environment tag that Traces should be tagged with.
  ## If not set the value will be inherited, in order, from the top level

~Output Truncated~

[root@appserver ~]# systemctl restart datadog-agent
[root@appserver ~]#

[root@appserver ~]# DD_AGENT_VERSION=latest
curl -L https://dtdg.co/latest-java-tracer -o /etc/datadog-agent/dd-java-agent.jar
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   145  100   145    0     0    814      0 --:--:-- --:--:-- --:--:--   814
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100 31.2M  100 31.2M    0     0  15.4M      0  0:00:02  0:00:02 --:--:-- 23.8M
[root@appserver ~]#
[root@appserver ~]# ll /etc/datadog-agent/dd-java-agent.jar
-rw-r--r-- 1 root root 32729838 Sep 23 22:23 /etc/datadog-agent/dd-java-agent.jar
[root@appserver ~]#

[root@appserver ~]# vi /opt/tomcat/bin/setenv.sh
[root@appserver ~]# cat /opt/tomcat/bin/setenv.sh
~Output Truncated~

# Datadog APM Java Tracer settings
CATALINA_OPTS="$CATALINA_OPTS \
-javaagent:/etc/datadog-agent/dd-java-agent.jar \
-Ddd.service=spring-boot-app \
-Ddd.env=production \
-Ddd.version=1.0.0 \
-Ddd.agent.host=localhost \
-Ddd.trace.enabled=true"
[root@appserver ~]#
[root@appserver ~]# systemctl restart tomcat
[root@appserver ~]# 

[root@appserver ~]# vi /var/lib/pgsql/data/postgresql.conf
[root@appserver ~]# cat /var/lib/pgsql/data/postgresql.conf
...omitted...
# Collecting performance statistics setting
shared_preload_libraries = 'pg_stat_statements'

# Enable collection of execution statistics
track_activities = on

# Track statistics for tables and indexes
track_counts = on

# Record timing information for I/O operations
track_io_timing = on

# Log sample queries (useful for detecting slow queries)
log_min_duration_statement = 1000   # Log SQL statements that take longer than 1 second
[root@appserver ~]#
[root@appserver ~]# systemctl restart postgresql
[root@appserver ~]#

postgres=# create user datadog with password 'yourpassword';
CREATE ROLE
postgres=#

postgres=# CREATE SCHEMA datadog;
CREATE SCHEMA
postgres=#

postgres=# GRANT USAGE ON SCHEMA datadog TO datadog;
GRANT
postgres=#

postgres=# GRANT pg_monitor TO datadog;
GRANT
postgres=#

postgres=# CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
CREATE EXTENSION
postgres=#

postgres=# CREATE OR REPLACE FUNCTION datadog.explain_statement(
   l_query TEXT,
   OUT explain JSON
)
RETURNS SETOF JSON AS
$$
DECLARE
curs REFCURSOR;
plan JSON;

BEGIN
   OPEN curs FOR EXECUTE pg_catalog.concat('EXPLAIN (FORMAT JSON) ', l_query);
   FETCH curs INTO plan;
   CLOSE curs;
   RETURN QUERY SELECT plan;
END;
$$
LANGUAGE 'plpgsql'
RETURNS NULL ON NULL INPUT
SECURITY DEFINER;
CREATE FUNCTION
quiz=#

postgres-# \q
[postgres@quiz ~]$
[postgres@quiz ~]$ psql -h localhost -U datadog -d postgres -A -c "select * from pg_stat_database limit 1;"
Password for user datadog:
datid|datname|numbackends|xact_commit|xact_rollback|blks_read|blks_hit|tup_returned|tup_fetched|tup_inserted|tup_updated|tup_deleted|conflicts|temp_files|temp_bytes|deadlocks|checksum_failures|checksum_last_failure|blk_read_time|blk_write_time|session_time|active_time|idle_in_transaction_time|sessions|sessions_abandoned|sessions_fatal|sessions_killed|stats_reset
0||0|0|0|72|530420|201511|118897|3|0|0|0|0|0|0|||0|0|0|0|0|0|0|0|0|
(1 row)
[postgres@quiz ~]$
[postgres@quiz ~]$ psql -h localhost -U datadog -d postgres -A -c "select * from pg_stat_activity limit 1;"
Password for user datadog:
datid|datname|pid|leader_pid|usesysid|usename|application_name|client_addr|client_hostname|client_port|backend_start|xact_start|query_start|state_change|wait_event_type|wait_event|state|backend_xid|backend_xmin|query_id|query|backend_type
||216421||||||||2025-10-19 06:30:17.392103+09||||Activity|AutoVacuumMain||||||autovacuum launcher
(1 row)
[postgres@quiz ~]$