Expedition root directory keeps growing

[stevenjwilliams83]

expedition@pan-expedition:/home/userSpace$ ls -la

total 100

drwxr-xr-x 5 www-data www-data  4096 Apr 15 07:56 .

drwxr-xr-x 4 root     root      4096 Sep  1  2017 ..

drwxr-xr-x 3 www-data www-data  4096 Apr  5 10:01 devices

-rw-r--r-- 1 www-data www-data    82 Apr 15 07:56 environmentParameters.php

-rw-r--r-- 1 www-data www-data 64913 Apr 16 18:01 panReadOrders.log

drwxrw-rw- 3 www-data www-data  4096 Apr 12 13:38 projects

-rw-r--r-- 1 www-data www-data   598 May 20  2018 userDefinitions.php

drwxr-xr-x 3 root     root      4096 Apr 10 14:50 utils

-rw-r--r-- 1 www-data www-data    36 Aug 15  2018 .uuid4

expedition@pan-expedition:/home/userSpace$ su ?

No passwd entry for user '?'

expedition@pan-expedition:/home/userSpace$ su www-data projects

Password:

su: Authentication failure

expedition@pan-expedition:/home/userSpace$ projects

No command 'projects' found, did you mean:

Command 'projectx' from package 'project-x' (universe)

Command 'projectl' from package 'projectl' (universe)

Command 'project' from package 'turnin-ng' (universe)

projects: command not found

expedition@pan-expedition:/home/userSpace$ cd projects

-bash: cd: projects: Permission denied

expedition@pan-expedition:/home/userSpace$ su cd project

No passwd entry for user 'cd'

expedition@pan-expedition:/home/userSpace$ su cd projects

No passwd entry for user 'cd'

expedition@pan-expedition:/home/userSpace$

[sjanita]

from the cli you will need to issue the command:

expedition@Expedition:/home/userSpace$ sudo su

[sudo] password for expedition:

root@Expedition:/home/userSpace# whoami

root

root@Expedition:/home/userSpace#

Now that you are root you can cd into the projects directory and into each directory to check for backup files

root@Expedition:/home/userSpace# cd projects/

root@Expedition:/home/userSpace/projects# ls -al

total 60

drwxrw-rw- 15 www-data www-data 4096 Apr 18 11:41 .

drwxr-xr-x  5 www-data www-data 4096 Mar 25 15:52 ..

drwxr--r--  3 www-data www-data 4096 Mar 25 15:58 5250MLRE

drwxr--r--  4 www-data www-data 4096 Apr 17 01:02 850MachineLearning

root@Expedition:/home/userSpace/projects# cd 850MachineLearning/

root@Expedition:/home/userSpace/projects/850MachineLearning# ls -al

total 596

drwxr--r--  4 www-data www-data   4096 Apr 17 01:02 .

drwxrw-rw- 15 www-data www-data   4096 Apr 18 11:41 ..

drwxr--r--  2 www-data www-data   4096 Apr 16 11:32 Backups

-rw-r--r--  1 www-data www-data  53200 Apr  1 09:45 MT-850MachineLearning.xml.dat

-rw-r--r--  1 www-data www-data  20752 Apr  1 09:45 MT-850MachineLearning.zip.dat

-rw-r--r--  1 www-data www-data  83680 Mar 27 11:31 PA-850_011901002830.xml_2019.03.27_11h32m.dat

-rw-r--r--  1 www-data www-data  80912 Mar 25 15:58 PA-850_011901002830.xml.dat

-rw-r--r--  1 www-data www-data 129424 Mar 28 10:17 Panorama-QA_007307000481.xml_2019.03.28_10h17m.dat

-rw-r--r--  1 www-data www-data 129360 Mar 28 09:16 Panorama-QA_007307000481.xml.dat

-rw-r--r--  1 www-data www-data  84736 Mar 27 11:36 panos_8.xml.dat

drwxr-xr-x  2 www-data www-data   4096 Mar 28 10:17 toImport

root@Expedition:/home/userSpace/projects/850MachineLearning# cd Backups/

root@Expedition:/home/userSpace/projects/850MachineLearning/Backups# ls -al

total 93792

drwxr--r-- 2 www-data www-data    4096 Apr 16 11:32 .

drwxr--r-- 4 www-data www-data    4096 Apr 17 01:02 ..

-rw-r--r-- 1 www-data www-data 4593712 Mar 27 04:19 base-3-27_MT_2019-03-27_10-19-31.sql.dat

-rw-r--r-- 1 www-data www-data 6011680 Apr 11 02:11 current_MT_2019-04-11_02-11-51.sql.dat

-rw-r--r-- 1 www-data www-data 1252368 Mar 25 16:01 undo_MT_2019-03-25_22-01-27.sql.dat

-rw-r--r-- 1 www-data www-data 4237120 Mar 25 17:16 undo_MT_2019-03-25_23-16-57.sql.dat

-rw-r--r-- 1 www-data www-data 4429968 Mar 27 04:05 undo_MT_2019-03-27_04-05-24.sql.dat

-rw-r--r-- 1 www-data www-data 4361488 Mar 27 04:00 undo_MT_2019-03-27_10-00-00.sql.dat

-rw-r--r-- 1 www-data www-data 4593712 Mar 27 04:19 undo_MT_2019-03-27_10-19-32.sql.dat

-rw-r--r-- 1 www-data www-data 4978240 Apr  1 09:39 undo_MT_2019-04-01_09-39-19.sql.dat

-rw-r--r-- 1 www-data www-data 6014112 Apr 10 13:34 undo_MT_2019-04-10_11-34-19.sql.dat

-rw-r--r-- 1 www-data www-data 6014256 Apr 11 02:04 undo_MT_2019-04-11_00-04-53.sql.dat

-rw-r--r-- 1 www-data www-data 6011856 Apr 11 02:12 undo_MT_2019-04-11_00-12-25.sql.dat

-rw-r--r-- 1 www-data www-data 6013072 Apr 11 02:12 undo_MT_2019-04-11_00-12-51.sql.dat

-rw-r--r-- 1 www-data www-data 6013248 Apr 11 02:13 undo_MT_2019-04-11_00-12-59.sql.dat

-rw-r--r-- 1 www-data www-data 6011680 Apr 11 02:13 undo_MT_2019-04-11_00-13-27.sql.dat

-rw-r--r-- 1 www-data www-data 6011824 Apr 11 02:13 undo_MT_2019-04-11_00-13-33.sql.dat

-rw-r--r-- 1 www-data www-data 6011952 Apr 11 02:20 undo_MT_2019-04-11_00-20-38.sql.dat

-rw-r--r-- 1 www-data www-data 6718736 Apr 15 04:24 undo_MT_2019-04-15_04-24-20.sql.dat

-rw-r--r-- 1 www-data www-data 6719552 Apr 16 11:32 undo_MT_2019-04-16_11-32-55.sql.dat

root@Expedition:/home/userSpace/projects/850MachineLearning/Backups#

[stevenjwilliams83]

so what is removable?

root@pan-expedition:/home/userSpace/projects/CoreMigration/Backups# ls -la

total 107556

drwxr--r-- 2 www-data www-data     4096 Apr 16 18:04 .

drwxr--r-- 4 www-data www-data     4096 Apr 12 16:02 ..

-rw-r--r-- 1 www-data www-data  1312928 Apr 12 13:41 4-12-2019_MT_2019-04-12_13-41-27.sql.dat

-rw-r--r-- 1 www-data www-data 45896112 Apr 16 18:04 4-16-2019_MT_2019-04-16_18-04-24.sql.dat

-rw-r--r-- 1 www-data www-data  1312928 Apr 12 13:41 undo_MT_2019-04-12_13-41-27.sql.dat

-rw-r--r-- 1 www-data www-data  1454112 Apr 12 16:01 undo_MT_2019-04-12_16-01-21.sql.dat

-rw-r--r-- 1 www-data www-data  1454272 Apr 12 16:01 undo_MT_2019-04-12_16-01-31.sql.dat

-rw-r--r-- 1 www-data www-data 12779920 Apr 16 18:01 undo_MT_2019-04-16_18-01-33.sql.dat

-rw-r--r-- 1 www-data www-data 45896112 Apr 16 18:04 undo_MT_2019-04-16_18-04-25.sql.dat

root@pan-expedition:/home/userSpace/projects/CoreMigration/Backups#

[sjanita]

you will want to remove the newer files first. 

But looking at the size of the files, these specific files are MB in size so I don't know how much of an impact they will have to free up disk space, but if those files are not needed they can be deleted.

[sjanita]

it also look as if you expenaded the size of the default partiion and also added a second disk. 

were you priginally storing traffic logs in the /PALogs directory? If yes have you checked that directory for any traffic log files that can be deleted?

[stevenjwilliams83]

No I am exporting the logs to the second HDD that I created. there is nothing in the PALogs directory. I cant get the database started either. 

[sjanita]

the DB cannot start likely due to the lack of storage left in the primary HDD.

if you are free for a zoom conference monday or tuesday afternoon feel free to send an email to fwmigrate (at) paloaltonetworks.com

[stevenjwilliams83]

HD is good on the root. 

expedition@pan-expedition:~$ systemctl status mariadb.service

● mariadb.service - MariaDB 10.1.38 database server

   Loaded: loaded (/lib/systemd/system/mariadb.service; enabled; vendor preset: enabled)

  Drop-In: /etc/systemd/system/mariadb.service.d

           └─migrated-from-my.cnf-settings.conf

   Active: failed (Result: exit-code) since Tue 2019-04-23 18:47:13 CDT; 4s ago

     Docs: man:mysqld(8)

           https://mariadb.com/kb/en/library/systemd/

  Process: 1549 ExecStart=/usr/sbin/mysqld $MYSQLD_OPTS $_WSREP_NEW_CLUSTER $_WSREP_START_POSITION (code=exited, status=1/FAILURE)

  Process: 1186 ExecStartPre=/bin/sh -c [ ! -e /usr/bin/galera_recovery ] && VAR= ||   VAR=`/usr/bin/galera_recovery`; [ $? -eq 0 ]   && systemctl set-environment _WSREP_START_POSITION=$VAR |

  Process: 1180 ExecStartPre=/bin/sh -c systemctl unset-environment _WSREP_START_POSITION (code=exited, status=0/SUCCESS)

  Process: 1159 ExecStartPre=/usr/bin/install -m 755 -o mysql -g root -d /var/run/mysqld (code=exited, status=0/SUCCESS)

Main PID: 1549 (code=exited, status=1/FAILURE)

   Status: "MariaDB server is down"

Apr 23 18:47:09 pan-expedition mysqld[1549]: 2019-04-23 18:47:09 140206857144576 [Note] InnoDB:  Percona XtraDB (http://www.percona.com) 5.6.42-84.2 started; log sequence number 247797360

Apr 23 18:47:10 pan-expedition mysqld[1549]: 2019-04-23 18:47:10 140205893871360 [Note] InnoDB: Dumping buffer pool(s) not yet started

Apr 23 18:47:10 pan-expedition mysqld[1549]: 2019-04-23 18:47:10 140206857144576 [Note] Plugin 'FEEDBACK' is disabled.

Apr 23 18:47:10 pan-expedition mysqld[1549]: 2019-04-23 18:47:10 140206857144576 [Note] Recovering after a crash using tc.log

Apr 23 18:47:10 pan-expedition mysqld[1549]: 2019-04-23 18:47:10 140206857144576 [ERROR] Can't init tc log

Apr 23 18:47:10 pan-expedition mysqld[1549]: 2019-04-23 18:47:10 140206857144576 [ERROR] Aborting

Apr 23 18:47:13 pan-expedition systemd[1]: mariadb.service: Main process exited, code=exited, status=1/FAILURE

Apr 23 18:47:13 pan-expedition systemd[1]: Failed to start MariaDB 10.1.38 database server.

Apr 23 18:47:13 pan-expedition systemd[1]: mariadb.service: Unit entered failed state.

Apr 23 18:47:13 pan-expedition systemd[1]: mariadb.service: Failed with result 'exit-code'.

lines 1-24/24 (END)

expedition@pan-expedition:~$ df -h

Filesystem                       Size  Used Avail Use% Mounted on

udev                             2.9G     0  2.9G   0% /dev

tmpfs                            595M  8.3M  587M   2% /run

/dev/mapper/Expedition--vg-root  109G  4.3G  100G   5% /

tmpfs                            3.0G     0  3.0G   0% /dev/shm

tmpfs                            5.0M     0  5.0M   0% /run/lock

tmpfs                            3.0G     0  3.0G   0% /sys/fs/cgroup

/dev/sdb                          99G   94G     0 100% /hdd

/dev/sda1                        472M  109M  340M  25% /boot

tmpfs                            595M     0  595M   0% /run/user/1000

expedition@pan-expedition:~$

[dgildelaig]

You have two issues here.

One is that you are running out of space in your system, and you should try to find out what is taking this space.

You already started to find out this process.

The other issue, which may have arose due to the lack of space, is that mariadb crashed and left the tc.log in an inconsistent state.

Check this thread

https://live.paloaltonetworks.com/t5/Expedition-Discussions/Incorrect-User-or-Password-from-GUI/td-p...

with the last posts, that it would tell you how to delete the tc.log file and restart the mariadb server again.

Anyhow, do not forget to keep with the first task, to make sure that you won't run out of space again soon.

[stevenjwilliams83]

Looks like even though I have the logs exporting to another hard drive, and have the temporary data structure folder also set to that other hard drive, expedition is still either moving or copying log files to the /data directory. Why is this?

root@pan-expedition:/# for i in `ls`; do du -hs $i; done

16M bin

106M boot

82G data

4.0K data2

12M datastore

0 dev

17M etc

52G hdd

255M home

0 initrd.img

0 initrd.img.old

710M lib

4.0K lib64

4.0K logs

16K lost+found

8.0K media

4.0K mnt

453M opt

16K PALogs

du: cannot access 'proc/71855/task/71855/fd/4': No such file or directory

du: cannot access 'proc/71855/task/71855/fdinfo/4': No such file or directory

du: cannot access 'proc/71855/fd/3': No such file or directory

du: cannot access 'proc/71855/fdinfo/3': No such file or directory

0 proc

43M root

31M run

14M sbin

8.0K snap

4.0K srv

0 sys

272K tmp

1.9G usr

958M var

0 vmlinuz

0 vmlinuz.old

root@pan-expedition:/# cd data

root@pan-expedition:/data# for i in `ls`; do du -hs $i; done

82G 10.20.63.85

root@pan-expedition:/data# cd 10.20.63.85

root@pan-expedition:/data/10.20.63.85# ls -la

total 85073012

drwx------ 2 root     root            4096 Apr 29 00:00 .

drwxrwxrwx 3 www-data www-data        4096 Apr  9 15:24 ..

-rw-r--r-- 1 root     root      2928389732 Apr 23 23:59 PA-5250-LOANER_traffic_2019_04_23_last_calendar_day.csv

-rw-r--r-- 1 root     root     16263451665 Apr 24 23:59 PA-5250-LOANER_traffic_2019_04_24_last_calendar_day.csv

-rw-r--r-- 1 root     root     16577735082 Apr 25 23:59 PA-5250-LOANER_traffic_2019_04_25_last_calendar_day.csv

-rw-r--r-- 1 root     root     15462734094 Apr 26 23:59 PA-5250-LOANER_traffic_2019_04_26_last_calendar_day.csv

-rw-r--r-- 1 root     root     19307974444 Apr 27 23:59 PA-5250-LOANER_traffic_2019_04_27_last_calendar_day.csv

-rw-r--r-- 1 root     root     12322880043 Apr 28 23:59 PA-5250-LOANER_traffic_2019_04_28_last_calendar_day.csv

-rw-r--r-- 1 root     root      4251551633 Apr 29 07:58 PA-5250-LOANER_traffic_2019_04_29_last_calendar_day.csv

root@pan-expedition:/data/10.20.63.85# ls -la

total 82256368

drwx------ 2 root     root            4096 Apr 29 08:02 .

drwxrwxrwx 3 www-data www-data        4096 Apr  9 15:24 ..

-rw-r--r-- 1 root     root     16263451665 Apr 24 23:59 PA-5250-LOANER_traffic_2019_04_24_last_calendar_day.csv

-rw-r--r-- 1 root     root     16577735082 Apr 25 23:59 PA-5250-LOANER_traffic_2019_04_25_last_calendar_day.csv

-rw-r--r-- 1 root     root     15462734094 Apr 26 23:59 PA-5250-LOANER_traffic_2019_04_26_last_calendar_day.csv

-rw-r--r-- 1 root     root     19307974444 Apr 27 23:59 PA-5250-LOANER_traffic_2019_04_27_last_calendar_day.csv

-rw-r--r-- 1 root     root     12322880043 Apr 28 23:59 PA-5250-LOANER_traffic_2019_04_28_last_calendar_day.csv

-rw-r--r-- 1 root     root      4295702848 Apr 29 08:02 PA-5250-LOANER_traffic_2019_04_29_last_calendar_day.csv

root@pan-expedition:/data/10.20.63.85#