Changing Prometheus TSDB directory

Written 2024-11-02. 186 words, about 1 minutes.

Stop prometheus and take backup

systemctl stop prometheus
sudo cp -vr /var/lib/prometheus /opt/intersec/prometheus_backup

Create new data directory

chown -R prometheus:prometheus /data
sudo -u prometheus mkdir /data/prometheus

Change prometheus service to use new directory

vim /etc/systemd/system/prometheus.service

[Unit]
Description=Prometheus
After=network-online.target
Requires=local-fs.target
After=local-fs.target

[Service]
Type=simple
Environment="GOMAXPROCS=8"
User=prometheus
Group=prometheus
ExecReload=/bin/kill -HUP $MAINPID
ExecStart=/usr/local/bin/prometheus \
  --storage.tsdb.path=/data/prometheus \  <---------- CHANGE DIRECTORY HERE
  --storage.tsdb.retention.time=30d \
  --storage.tsdb.retention.size=0 \
  --web.config.file=/etc/prometheus/web.yml \
  --web.console.libraries=/etc/prometheus/console_libraries \
  --web.console.templates=/etc/prometheus/consoles \
  --web.listen-address=0.0.0.0:9090 \
  --web.external-url= \
  --config.file=/etc/prometheus/prometheus.yml

CapabilityBoundingSet=CAP_SET_UID
LimitNOFILE=65000
LockPersonality=true
NoNewPrivileges=true
MemoryDenyWriteExecute=true
PrivateDevices=true
PrivateTmp=true
ProtectHome=true
RemoveIPC=true
RestrictSUIDSGID=true
#SystemCallFilter=@signal @timer

ReadWritePaths=/data/prometheus   <--------------- CHANGE DIRECTORY HERE

PrivateUsers=true
ProtectControlGroups=true
ProtectKernelModules=true
ProtectKernelTunables=true
ProtectSystem=strict


SyslogIdentifier=prometheus
Restart=always
TimeoutStopSec=600s

[Install]
WantedBy=multi-user.target

Verify permissions and owner

ls -al /data/
ls -al /data/prometheus/
root@dc2-monitoring-server-1:~# ls -al /data/prometheus/
total 120
drwxr-xr-x 25 prometheus prometheus  4096  4 oct.  09:54 .
drwxr-xr-x  4 prometheus prometheus  4096  4 oct.  09:53 ..

Move the TSDB to new location

mv /var/lib/prometheus/* /data/prometheus/

Reload service file and start prometheus, verify if all is ok with journalctl and df

systemctl daemon-reload
systemctl start prometheus
journalctl -xe
df -h

Back to the list of posts