summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorValerii Ponomarov <vponomar@redhat.com>2019-02-18 19:35:51 +0530
committervponomar <vponomar@redhat.com>2019-03-18 10:47:12 +0000
commit0fcdb081517c5904969b89b20326d21b361e448e (patch)
tree9a9608ee930b6334d660ee4e2c910f8fb251ee6d
parent46bf49b318a25f8edf2ad6e38aebc99871552b9b (diff)
[Deployment] Add playbook for gathering logs from cluster nodes
For case when it is needed to provide info for bugreport or debug in general. It can be run anytime, separately from the deployment logic. Example command for running this new playbook: $ tox -e ocp3.11 -- ansible-playbook -i 127.0.0.1, \ playbooks/gather_logs.yaml \ -e config_filepath=/path/to/the/cns-automation-config.yaml \ -e output_artifacts_dir=../cluster_logs_ocp310_ocs311/ NOTE: 'tox -e ocp3.11' can be omitted only in case you enable 'yedit' (yaml edit) module by other means. Change-Id: Icb6bf442c07d8d949c930747e8131231a2ff112c
-rw-r--r--deployment/playbooks/gather_logs.yaml883
-rw-r--r--deployment/playbooks/get_ocp_info.yaml2
-rw-r--r--deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml5
3 files changed, 890 insertions, 0 deletions
diff --git a/deployment/playbooks/gather_logs.yaml b/deployment/playbooks/gather_logs.yaml
new file mode 100644
index 00000000..33b9114f
--- /dev/null
+++ b/deployment/playbooks/gather_logs.yaml
@@ -0,0 +1,883 @@
+# Expected vars:
+#
+# - 'config_filepath' - required. It is expected to be provided and must
+# store filepath for the config file used by automated test caes.
+# Current playbook will take all the nodes info from it.
+#
+# - 'output_artifacts_dir' - optional. It is directory where should be saved
+# generated/gathered files.
+#
+# Command to run this playbook:
+#
+# $ tox -e ocp3.11 -- ansible-playbook -i 127.0.0.1, \
+# playbooks/gather_logs.yaml \
+# -e config_filepath=/path/to/the/cns-automation-config.yaml \
+# -e output_artifacts_dir=../cluster_logs/
+
+---
+- hosts: localhost
+ connection: local
+ gather_facts: no
+ tasks:
+ - name: Process config file and find all the nodes of an OpenShift cluster
+ command: "python -c \"import yaml ;
+ config = yaml.load(open('{{ config_filepath }}', 'r'));
+ print(' '.join(list(config['ocp_servers']['master'].keys())));
+ print(' '.join(list(config['ocp_servers']['nodes'].keys())));
+ print(' '.join(list(config['gluster_servers'].keys())));
+ print(config.get('openshift', config.get('cns',
+ {}))['heketi_config']['heketi_server_url']);
+ print(config.get('openshift', config.get('cns',
+ {}))['heketi_config']['heketi_cli_user']);
+ print(config.get('openshift', config.get('cns',
+ {}))['heketi_config']['heketi_cli_key'])\""
+ register: config_data
+ - debug:
+ msg: "{{ config_data }}"
+ - name: Save config data to the host vars
+ set_fact:
+ master_nodes: "{{ config_data.stdout_lines[0].split(' ') }}"
+ compute_nodes: "{{ config_data.stdout_lines[1].split(' ') }}"
+ gluster_nodes: "{{ config_data.stdout_lines[2].split(' ') }}"
+ heketi_server_url: "{{ config_data.stdout_lines[3] }}"
+ heketi_cli_user: "{{ config_data.stdout_lines[4] }}"
+ heketi_cli_key: "{{ config_data.stdout_lines[5] }}"
+ - name: Print list of master nodes IP addresses
+ debug:
+ msg: "{{ master_nodes }}"
+ - name: Print list of compute nodes IP addresses
+ debug:
+ msg: "{{ compute_nodes }}"
+ - name: Print list of gluster nodes IP addresses
+ debug:
+ msg: "{{ gluster_nodes }}"
+ - name: Add gathered master IP addresses to the Ansible host list
+ add_host:
+ hostname: "{{ item }}"
+ ansible_host: "{{ item }}"
+ groups: logs_ocp_nodes, logs_single_master_node, logs_master_nodes
+ with_items: "{{ master_nodes }}"
+ - name: Add gathered compute IP addresses to the Ansible host list
+ add_host:
+ hostname: "{{ item }}"
+ ansible_host: "{{ item }}"
+ groups: logs_ocp_nodes, logs_compute_nodes
+ with_items: "{{ compute_nodes }}"
+ - name: Add gathered gluster IP addresses to the Ansible host list
+ add_host:
+ hostname: "{{ item }}"
+ ansible_host: "{{ item }}"
+ groups: logs_ocp_nodes, logs_gluster_nodes
+ with_items: "{{ gluster_nodes }}"
+
+- hosts: all
+ gather_facts: no
+ tasks:
+ - set_fact:
+ output_artifacts_dir: "{{
+ (output_artifacts_dir | default('../cluster_logs/')) }}"
+ - set_fact:
+ output_artifacts_dir: "{{ output_artifacts_dir
+ }}{% if output_artifacts_dir[-1] != '/' %}{{ '/' }}{% endif %}"
+ - name: Get hostname of the current host
+ shell: "hostname"
+ register: hostname
+ - name: Save hostname to the var
+ set_fact:
+ current_hostname: "{{ hostname.stdout_lines[0].strip() }}"
+ separator_line: "{{ '=' * 79 }}"
+ - name: Create output artifacts directory if absent
+ delegate_to: localhost
+ file:
+ path: "{{ output_artifacts_dir }}"
+ state: directory
+ run_once: yes
+ - name: Install 'rsync' package which is required by 'synchronize' module
+ yum:
+ name: rsync
+ state: installed
+ ignore_errors: yes
+
+# All nodes SOS reports
+- hosts: logs_ocp_nodes
+ gather_facts: no
+ tasks:
+ - name: Install 'sos' package if absent
+ package:
+ name: sos
+ state: present
+ - name: Run sosreport command
+ shell: "sosreport --batch --verbose --tmp-dir . --label customuniquelabel"
+ register: sosreport_output
+ - name: Print the output of the sosreport command
+ debug:
+ msg: "{{ sosreport_output }}"
+ - name: Get name of the generated sos-file
+ shell: 'echo -e "{{ sosreport_output.stdout }}" | grep customuniquelabel'
+ register: sos_filepath
+ - name: Copy generated files to the localhost
+ fetch:
+ src: "{{ sos_filepath.stdout_lines[0].strip() }}"
+ dest: "{{ output_artifacts_dir }}sosreports/"
+ flat: yes
+ fail_on_missing: yes
+
+- hosts: logs_single_master_node
+ gather_facts: no
+ vars:
+ master_package_list:
+ - docker
+ - heketi
+ master_service_list:
+ - docker
+ - multipathd
+ heketi_pod_package_list:
+ - gluster
+ - heketi
+ gluster_pod_package_list:
+ - gluster
+ - heketi
+ - targetcli
+ - gluster-block
+ - tcmu-runner
+ - python-configshell
+ - python-rtslib
+ gluster_pod_service_list:
+ - glusterd
+ - heketi
+ - gluster-blockd
+ - gluster-block-target
+ - tcmu-runner
+ - rpcbind
+ tasks:
+
+ # Master node info
+ - name: Get distro version
+ shell: "uname -a"
+ register: master_linux_kernel_version
+ - name: Get Red Hat release info
+ shell: "cat /etc/redhat-release"
+ register: master_rh_release
+ - name: Create grep filter with all the packages we are interested in
+ set_fact:
+ package_filter: "{{ package_filter | default('grep') + ' -e ' + item }}"
+ with_items: "{{ master_package_list }}"
+ - name: Get list of installed packages we are interested in
+ shell: "rpm -qa | {{ package_filter }}"
+ register: master_packages
+ - name: Get status of services on OCP Master node
+ shell: "systemctl list-units {{ master_service_list | join('.service ') }}.service
+ --type=service --all --no-pager --no-legend"
+ register: master_services
+ - name: Get OpenShift client version
+ shell: "(oc version | grep -e 'oc ' -e 'openshift' -e 'kube') ||
+ echo failed_to_get_oc_version_info"
+ register: master_oc_version
+ - name: Get list of OCP nodes
+ shell: "oc get nodes -o wide || echo failed_to_get_list_of_nodes"
+ register: master_ocp_nodes
+ - name: Get info about all the docker images used in OCP cluster
+ shell: "(oc get pods --all-namespaces
+ -o=custom-columns=:.status.containerStatuses[*].image | grep -v -e '^$' | uniq) ||
+ echo failed_to_get_list_of_images"
+ register: master_image_info
+ - name: Write master data to the data file
+ delegate_to: localhost
+ yedit:
+ src: "{{ output_artifacts_dir }}master_data.yaml"
+ state: present
+ edits:
+ - key: master
+ value:
+ Linux kernel version: "{{ master_linux_kernel_version.stdout_lines }}"
+ Red Hat release info: "{{ master_rh_release.stdout_lines }}"
+ List of Packages: "{{ master_packages.stdout_lines }}"
+ List of services: "{{ master_services.stdout_lines }}"
+ OC Version: "{{ master_oc_version.stdout_lines }}"
+ OCP nodes: "{{ master_ocp_nodes.stdout_lines }}"
+ Images info: "{{ master_image_info.stdout_lines }}"
+
+ # Heketi POD logs, config and DB dump
+ - name: Get heketi POD
+ shell: "oc get pods --all-namespaces -l heketi --no-headers
+ -o=custom-columns=:.metadata.name,:.metadata.namespace"
+ register: heketi_pods
+ retries: 10
+ delay: 6
+ until: heketi_pods is succeeded
+ - name: DEBUG HEKETI
+ debug:
+ msg: "{{ heketi_pods }}"
+ - block:
+ - name: Create var with destination dir path
+ set_fact:
+ dir_path: "{{ output_artifacts_dir }}heketi_pod/"
+ - name: Create compute directory if absent
+ delegate_to: localhost
+ file:
+ path: "{{ dir_path }}"
+ state: directory
+ - name: Set Heketi POD name and Heketi namespace as vars
+ set_fact:
+ heketi_pod_name: "{{
+ (heketi_pods.stdout_lines[0].split(' ') | list)[0] }}"
+ heketi_pod_namespace: "{{
+ (heketi_pods.stdout_lines[0].split(' ') | list)[-1] }}"
+ - name: Set Heketi pod command prefix
+ set_fact:
+ heketi_pod_cmd_prefix: "oc exec {{ heketi_pod_name
+ }} --namespace {{ heketi_pod_namespace }} --"
+ - name: Get the Heketi config from the Heketi POD
+ shell: 'echo -e "$({{ heketi_pod_cmd_prefix
+ }} cat /etc/heketi/heketi.json ||
+ echo error_failed_to_get_the_heketi_config_file)" |
+ tee /tmp/heketi_config.json'
+ - name: Copy the Heketi config
+ fetch:
+ src: "/tmp/heketi_config.json"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+ - name: Save Heketi POD logs
+ shell: "(oc logs {{ heketi_pod_name }} ||
+ echo 'ERROR! Failed to get the Heketi logs.') > /tmp/heketi.log"
+ - name: Copy the Heketi logs
+ fetch:
+ src: "/tmp/heketi.log"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+ - name: Dump the Heketi DB
+ shell: 'echo -e "$({{ heketi_pod_cmd_prefix }} heketi-cli --server {{
+ hostvars["localhost"]["heketi_server_url"] }} --user {{
+ hostvars["localhost"]["heketi_cli_user"] }} --secret {{
+ hostvars["localhost"]["heketi_cli_key"]
+ }} db dump ||
+ echo \{\"error\"\:\"failed_to_get_the_Heketi_db_dump\"\})" |
+ python -m json.tool > /tmp/heketi_db_dump.json'
+ - name: Copy the Heketi DB dump
+ fetch:
+ src: "/tmp/heketi_db_dump.json"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+ - name: Get storage release version from Heketi POD
+ shell: "{{ heketi_pod_cmd_prefix }} cat /etc/redhat-storage-release ||
+ echo failed_to_read_redhat_storage_release_info"
+ register: heketi_pod_storage_release_version
+ - name: Get info about packages on Heketi POD
+ shell: "{{ heketi_pod_cmd_prefix }} rpm -qa | grep -e {{
+ heketi_pod_package_list | join(' -e ') }} ||
+ echo failed_to_read_list_of_installed_packages"
+ register: heketi_pod_packages
+ - name: Write Heketi data to the data file
+ delegate_to: localhost
+ yedit:
+ src: "{{ dir_path }}heketi_pod_data.yaml"
+ state: present
+ edits:
+ - key: "Storage_release_version"
+ value: "{{ heketi_pod_storage_release_version.stdout_lines }}"
+ - key: "List_of_Packages"
+ value: "{{ heketi_pod_packages.stdout_lines }}"
+ when: "((heketi_pods.stdout_lines | join('')).strip() | length) > 0"
+
+ # Gluster PODs
+ - name: Get list of Gluster PODs
+ shell: "oc get pods --all-namespaces -l glusterfs-node --no-headers
+ -o=custom-columns=:.metadata.name,:.metadata.namespace"
+ register: gluster_pods
+ retries: 10
+ delay: 6
+ until: gluster_pods is succeeded
+ - name: DEBUG GLUSTER
+ debug:
+ msg: "{{ gluster_pods }}"
+ - name: Create var describing the Gluster cluster deployment type
+ set_fact:
+ is_gluster_containerized: "{{
+ ((gluster_pods.stdout_lines | join('')).strip() | length) > 0 }}"
+ delegate_to: "{{ item }}"
+ delegate_facts: yes
+ with_items: "{{ groups['all'] }}"
+ - block:
+ - name: Create var with destination dir path
+ set_fact:
+ dir_path: "{{ output_artifacts_dir }}gluster_pods/"
+ - name: Create directory if absent
+ delegate_to: localhost
+ file:
+ path: "{{ dir_path }}"
+ state: directory
+ - name: Define storage namespace
+ set_fact:
+ storage_namespace: "{{ (gluster_pods.stdout_lines[0].split(
+ ' ') | list)[-1] }}"
+ - name: Define gluster POD names
+ set_fact:
+ gluster_pod_names: "{{ (gluster_pod_names | default([])) +
+ [(item.split(' ') | list)[0]] }}"
+ with_items: "{{ gluster_pods.stdout_lines[0:] }}"
+ - debug:
+ msg: "{{ gluster_pod_names }}"
+
+ - name: Get storage release version from Gluster PODs
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} --
+ cat /etc/redhat-storage-release) ||
+ echo failed_to_get_redhat_storage_release_info"
+ with_items: "{{ gluster_pod_names }}"
+ register: gluster_pod_storage_release_version_results
+ - name: Process gluster PODs storage release versions results
+ set_fact:
+ gluster_pod_storage_release_version_processed: "{{
+ gluster_pod_storage_release_version_processed | default({}) | combine(
+ {(item.item.strip().split(' ')[0]): item.stdout_lines},
+ recursive=True
+ ) }}"
+ with_items: "{{ gluster_pod_storage_release_version_results.results }}"
+ - name: Get info about packages on Gluster PODs
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} --
+ rpm -qa | grep -e {{ gluster_pod_package_list | join(' -e ') }}) ||
+ echo failed_to_get_packages_info_from_gluster_pod"
+ with_items: "{{ gluster_pod_names }}"
+ register: gluster_pod_package_list_results
+ - name: Process gluster PODs package lists results
+ set_fact:
+ gluster_pod_package_list_processed: "{{
+ gluster_pod_package_list_processed | default({}) | combine(
+ {(item.item.strip().split(' ')[0]): item.stdout_lines},
+ recursive=True
+ ) }}"
+ with_items: "{{ gluster_pod_package_list_results.results }}"
+ - name: Get info about services on Gluster PODs
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} --
+ systemctl list-units {{
+ gluster_pod_service_list | join('.service ') }}.service
+ --type=service --all --no-pager --no-legend) ||
+ echo failed_to_get_services_info_from_gluster_pod"
+ with_items: "{{ gluster_pod_names }}"
+ register: gluster_pod_service_list_results
+ - name: Process gluster PODs service lists results
+ set_fact:
+ gluster_pod_service_list_processed: "{{
+ gluster_pod_service_list_processed | default({}) | combine(
+ {(item.item.strip().split(' ')[0]): item.stdout_lines},
+ recursive=True
+ ) }}"
+ with_items: "{{ gluster_pod_service_list_results.results }}"
+ - name: Write Gluster PODs data to the data file
+ delegate_to: localhost
+ yedit:
+ src: "{{ dir_path }}gluster_pods_packages_and_services_data.yaml"
+ state: present
+ edits:
+ - key: gluster_pods
+ value:
+ Storage release version: "{{
+ gluster_pod_storage_release_version_processed }}"
+ List of Packages: "{{ gluster_pod_package_list_processed }}"
+ List of Services: "{{ gluster_pod_service_list_processed }}"
+
+ - name: Get 'targetcli ls' output
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} --
+ targetcli ls) || echo failed_to_get_targetcli_ls_output"
+ with_items: "{{ gluster_pod_names }}"
+ register: gluster_pod_targetcli_ls_results
+ - debug:
+ msg: "{{ gluster_pod_targetcli_ls_results }}"
+ - name: Write Gluster PODs data to the data file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-targetcli-ls"
+ with_items: "{{ gluster_pod_targetcli_ls_results.results }}"
+
+ - name: Create gluster log directories on the master node
+ file:
+ path: "/tmp/gluster/{{ item }}-var_log_glusterfs"
+ state: directory
+ with_items: "{{ gluster_pod_names }}"
+ - name: Copy '/var/log/glusterfs/*' files to the master
+ shell: "(oc cp {{ storage_namespace }}/{{ item }}:/var/log/glusterfs
+ /tmp/gluster/{{ item }}-var_log_glusterfs) ||
+ echo failed_to_copy_var_log_glusterfs_files"
+ with_items: "{{ gluster_pod_names }}"
+ - name: Copy '/var/log/glusterfs/*' files from the master to the localhost
+ synchronize:
+ src: "/tmp/gluster/{{ item }}-var_log_glusterfs/"
+ dest: "{{ dir_path }}{{ item }}-var_log_glusterfs/"
+ mode: pull
+ recursive: yes
+ use_ssh_args: yes
+ with_items: "{{ gluster_pod_names }}"
+ ignore_errors: yes
+
+ - name: Get 'dmesg -T' info
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- dmesg -T) || echo failed_toget_dmesg_-T_info"
+ with_items: "{{ gluster_pod_names }}"
+ register: gluster_pods_dmesg_results
+ - name: Write Gluster PODs dmesg data to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-dmesg"
+ with_items: "{{ gluster_pods_dmesg_results.results }}"
+
+ - name: Get list of processes and info for processes with 'D' stat
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} -- ps aux ;
+ oc exec {{ item }} --namespace {{ storage_namespace }} --
+ ps -aux | tee /dev/tty | awk {'if ( $8 ~ \"D\" ) print $2'} |
+ while read -r pid ;
+ do echo -e \"\nRunning '/proc/$pid/stack' command:\";
+ cat /proc/$pid/stack ;
+ done) || echo failed_to_get_info_about_processes_with_D_stat"
+ with_items: "{{ gluster_pod_names }}"
+ register: gluster_pod_io_processes_info
+ - name: Write Gluster PODs 'I\O' proccesses info to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-ps"
+ with_items: "{{ gluster_pod_io_processes_info.results }}"
+
+ - name: List dirs and files in '/sys/module/dm_multipath'
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- ls -l /sys/module/dm_multipath) ||
+ echo failed_to_list_files_in__sys_module_dm_multipath"
+ with_items: "{{ gluster_pod_names }}"
+ register: sys_module_dm_multipath_results
+ - name: Write Gluster PODs 'ls -l /sys/module/dm_multipath' output to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0]
+ }}-ls-sys_module_dm_multipath"
+ with_items: "{{ sys_module_dm_multipath_results.results }}"
+
+ - name: "Run 'lsmod | egrep target_core' command"
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- lsmod | egrep target_core) ||
+ echo failed_to_get_lsmod_info_for_target_core"
+ with_items: "{{ gluster_pod_names }}"
+ register: lsmod_target_core_results
+ - name: Write Gluster PODs 'lsmod | egrep target_core' command results to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{
+ (item.item.strip().split(' ') | list)[0] }}-lsmod_target_core"
+ with_items: "{{ lsmod_target_core_results.results }}"
+
+ - name: Get info about devices
+ shell: '(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- bash -c "echo -e \"{{ separator_line }}\nlsblk info:\"; lsblk;
+ echo -e \"{{ separator_line }}\nPVs info:\"; pvs;
+ echo -e \"{{ separator_line }}\nVGs info:\"; vgs;
+ echo -e \"{{ separator_line }}\nLVs info:\"; lvs;
+ echo -e \"{{ separator_line }}\"") ||
+ echo failed_to_get_list_of_pvs_vgs_and_lvs'
+ with_items: "{{ gluster_pod_names }}"
+ register: lsblk_pvs_vgs_lvs_results
+ - name: Write Gluster PODs info about PVs, VGs and LVs to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0]
+ }}-lsblk_pvs_vgs_lvs"
+ with_items: "{{ lsblk_pvs_vgs_lvs_results.results }}"
+
+ - name: Read 'journalctl' output
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- journalctl) || echo failed_to_read_journalctl_output"
+ with_items: "{{ gluster_pod_names }}"
+ register: journalctl_results
+ - name: Write Gluster PODs 'journalctl' output to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0]
+ }}-journalctl"
+ with_items: "{{ journalctl_results.results }}"
+
+ - name: Read 'mount' output
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- mount) || echo failed_to_read_mount_output"
+ with_items: "{{ gluster_pod_names }}"
+ register: mount_results
+ - name: Write Gluster PODs 'mount' output to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-mount"
+ with_items: "{{ mount_results.results }}"
+
+ - name: Create archive from ' /etc/target/' dir
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- tar -czvf etc_target.tar.gz /etc/target/ ) ||
+ echo failed_to_archive__etc_target_dir"
+ with_items: "{{ gluster_pod_names }}"
+ - name: Copy archive of the '/etc/target/' dir to the master
+ shell: "(oc cp {{ storage_namespace }}/{{ item }}:/etc_target.tar.gz
+ /tmp/gluster/{{ item }}-etc_target.tar.gz) ||
+ echo failed_to_copy_etc_target_file"
+ with_items: "{{ gluster_pod_names }}"
+ - name: Copy archive of the '/etc/target/' dir to the localhost
+ fetch:
+ src: "/tmp/gluster/{{ item }}-etc_target.tar.gz"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+ with_items: "{{ gluster_pod_names }}"
+
+ - name: Create archive from '/sys/kernel/config/target/' dir
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- tar -czvf sys_kernel_config_target.tar.gz /sys/kernel/config/target/ ) ||
+ echo failed_to_archive__sys_kernel_config_target_dir"
+ with_items: "{{ gluster_pod_names }}"
+ - name: Copy archive of the '/sys/kernel/config/target/' dir to the master
+ shell: "(oc cp {{ storage_namespace }}/{{ item }}:/sys_kernel_config_target.tar.gz
+ /tmp/gluster/{{ item }}-sys_kernel_config_target.tar.gz) ||
+ echo failed_to_copy_sys_kernel_config_target_file"
+ with_items: "{{ gluster_pod_names }}"
+ - name: Copy archive of the '/sys/kernel/config/target/' dir to the localhost
+ fetch:
+ src: "/tmp/gluster/{{ item }}-sys_kernel_config_target.tar.gz"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+ with_items: "{{ gluster_pod_names }}"
+ when: "is_gluster_containerized"
+
+# Gather info from gluster nodes in case of 'standalone' deployment
+- hosts: logs_gluster_nodes
+ gather_facts: no
+ vars:
+ gluster_package_list:
+ - gluster
+ - heketi
+ - targetcli
+ - gluster-block
+ - tcmu-runner
+ - python-configshell
+ - python-rtslib
+ gluster_service_list:
+ - glusterd
+ - heketi
+ - gluster-blockd
+ - gluster-block-target
+ - tcmu-runner
+ - rpcbind
+ tasks:
+ - block:
+ - name: Create var with destination dir path
+ set_fact:
+ dir_path: "{{ output_artifacts_dir }}gluster_nodes/"
+ - name: Create directory if absent
+ delegate_to: localhost
+ file:
+ path: "{{ dir_path }}"
+ state: directory
+ run_once: yes
+
+ - name: Get storage release version from Gluster node
+ shell: "(cat /etc/redhat-storage-release) ||
+ echo failed_to_get_redhat_storage_release_info"
+ register: gluster_node_storage_release_version_results
+ - name: Get info about packages on Gluster node
+ shell: "(rpm -qa | grep -e {{ gluster_package_list | join(' -e ') }}) ||
+ echo failed_to_get_packages_info_from_gluster_node"
+ register: gluster_node_package_list_results
+ - name: Get info about services on Gluster node
+ shell: "(systemctl list-units {{
+ gluster_service_list | join('.service ') }}.service
+ --type=service --all --no-pager --no-legend) ||
+ echo failed_to_get_services_info_from_gluster_node"
+ register: gluster_node_service_list_results
+ - name: Write Gluster node data to the data file
+ delegate_to: localhost
+ yedit:
+ src: "{{ dir_path }}gluster_nodes_packages_and_services_data.yaml"
+ state: present
+ edits:
+ - key: "gluster_node_{{ current_hostname }}"
+ value:
+ Storage release version: "{{
+ gluster_node_storage_release_version_results.stdout }}"
+ List of Packages: "{{ gluster_node_package_list_results.stdout_lines }}"
+ List of Services: "{{ gluster_node_service_list_results.stdout_lines }}"
+
+ - name: Get 'targetcli ls' output
+ shell: "targetcli ls || echo failed_to_get_targetcli_ls_output"
+ register: gluster_node_targetcli_ls_results
+ - name: Write Gluster node data to the data file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ gluster_node_targetcli_ls_results.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-targetcli-ls"
+
+ - name: Copy '/var/log/glusterfs/*' files from the current gluster node
+ synchronize:
+ src: "/var/log/glusterfs/"
+ dest: "{{ dir_path }}{{ current_hostname }}-var_log_glusterfs/"
+ mode: pull
+ recursive: yes
+ use_ssh_args: yes
+ ignore_errors: yes
+
+ - name: Get info about space usage
+ shell: '(echo -e "File system disk space usage on the {{
+ current_hostname }} node:\n"; df -Th) ||echo failed_to_get_df_info'
+ register: df_info
+ - name: Save mounts info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ df_info.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_df"
+
+ - name: Get 'dmesg -T' info
+ shell: "dmesg -T || echo failed_toget_dmesg_-T_info"
+ register: gluster_node_dmesg_results
+ - name: Write Gluster node dmesg data to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ gluster_node_dmesg_results.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-dmesg"
+
+ - name: Get list of processes and info for processes with 'D' stat
+ shell: "( ps aux ;
+ ps -aux | tee /dev/tty | awk {'if ( $8 ~ \"D\" ) print $2'} |
+ while read -r pid ;
+ do echo -e \"\nRunning '/proc/$pid/stack' command:\";
+ cat /proc/$pid/stack ;
+ done) || echo failed_to_get_info_about_processes_with_D_stat"
+ register: gluster_node_io_processes_info
+ - name: Write Gluster node 'I\O' proccesses info to a file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ gluster_node_io_processes_info.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-ps"
+
+ - name: List dirs and files in '/sys/module/dm_multipath'
+ shell: "ls -l /sys/module/dm_multipath ||
+ echo failed_to_list_files_in__sys_module_dm_multipath"
+ register: sys_module_dm_multipath_results
+ - name: Write Gluster node 'ls -l /sys/module/dm_multipath' output to a file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ sys_module_dm_multipath_results.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-ls-sys_module_dm_multipath"
+
+ - name: "Run 'lsmod | egrep target_core' command"
+ shell: "(lsmod | egrep target_core) ||
+ echo failed_to_get_lsmod_info_for_target_core"
+ register: lsmod_target_core_results
+ - name: Write Gluster node 'lsmod | egrep target_core' command results to a file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ lsmod_target_core_results.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-lsmod_target_core"
+
+ - name: Get info about devices
+ shell: '(echo -e "{{ separator_line }}\nlsblk info:"; lsblk;
+ echo -e "{{ separator_line }}\nPVs info:"; pvs;
+ echo -e "{{ separator_line }}\nVGs info:"; vgs;
+ echo -e "{{ separator_line }}\nLVs info:"; lvs;
+ echo -e "{{ separator_line }}\nll /dev/disk/by-path/ip-*:";
+ ll /dev/disk/by-path/ip-*; echo {{ separator_line }}) ||
+ echo failed_to_get_info'
+ register: lsblk_pvs_vgs_lvs
+ - name: Save devices info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ lsblk_pvs_vgs_lvs.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_lsblk_pvs_vgs_lvs"
+
+ - name: Read 'journalctl' output
+ shell: "journalctl || echo failed_to_read_journalctl_output"
+ register: journalctl_results
+ - name: Write Gluster node 'journalctl' output to a file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ journalctl_results.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-journalctl"
+
+ - name: Read 'mount' output
+ shell: "mount || echo failed_to_read_mount_output"
+ register: mount_results
+ - name: Write Gluster node 'mount' output to a file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ mount_results.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-mount"
+
+ - name: Create archive from ' /etc/target/' dir
+ shell: "tar -czvf /tmp/gluster/{{ current_hostname
+ }}-etc_target.tar.gz /etc/target/ ||
+ echo failed_to_archive__etc_target_dir"
+ - name: Copy archive of the '/etc/target/' dir to the localhost
+ fetch:
+ src: "/tmp/gluster/{{ current_hostname }}-etc_target.tar.gz"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+
+ - name: Create archive from '/sys/kernel/config/target/' dir
+ shell: "tar -czvf /tmp/gluster/{{ current_hostname
+ }}-sys_kernel_config_target.tar.gz /sys/kernel/config/target/ ||
+ echo failed_to_archive__sys_kernel_config_target_dir"
+ - name: Copy archive of the '/sys/kernel/config/target/' dir to the localhost
+ fetch:
+ src: "/tmp/gluster/{{ current_hostname }}-sys_kernel_config_target.tar.gz"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+
+ - name: Create archive from '/var/log/messages' dir
+ shell: "tar -czvf var_log_messages.tar.gz /var/log/messages"
+ retries: 15
+ delay: 2
+ register: result
+ until: result is succeeded
+ ignore_errors: yes
+ - name: Copy the archive to the localhost
+ fetch:
+ src: "var_log_messages.tar.gz"
+ dest: "{{ dir_path }}{{ current_hostname }}_var_log_messages.tar.gz"
+ flat: yes
+ fail_on_missing: yes
+ ignore_errors: yes
+ when: "not is_gluster_containerized"
+
+# Gather info from compute nodes
+- hosts: logs_compute_nodes
+ gather_facts: no
+ vars:
+ compute_package_list:
+ - docker
+ - heketi
+ - rpcbind
+ compute_service_list:
+ - docker
+ - multipathd
+ - rpcbind
+ - iscsid
+ tasks:
+ - name: Create var with destination dir path
+ set_fact:
+ dir_path: "{{ output_artifacts_dir }}compute_nodes/"
+ - name: Create compute directory if absent
+ delegate_to: localhost
+ file:
+ path: "{{ dir_path }}"
+ state: directory
+ run_once: yes
+
+ - name: Create grep filter with all the packages we are interested in
+ set_fact:
+ package_filter: "{{ package_filter | default('grep') + ' -e ' + item }}"
+ with_items: "{{ compute_package_list }}"
+ - name: Get list of installed packages we are interested in
+ shell: "rpm -qa | {{ package_filter }} || echo failed_to_get_info"
+ register: compute_packages
+ - name: Write compute nodes package list to a file
+ delegate_to: localhost
+ copy:
+ content: "{{ compute_packages.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_packages.yaml"
+
+ - name: Get status of services on OCP Compute nodes
+ shell: "systemctl list-units {{
+ compute_service_list | join('.service ') }}.service --no-pager
+ --type=service --all --no-legend || echo failed_to_get_info"
+ register: compute_services
+ - name: Write compute nodes service list to a file
+ delegate_to: localhost
+ copy:
+ content: "{{ compute_services.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_services.yaml"
+
+ - name: Get multipath info
+ shell: "(echo 'Multipath config is following:'; cat /etc/multipath.conf;
+ echo -e \"{{ separator_line }}\nResults of 'nmultipath -ll' command:\";
+ multipath -ll; echo {{ separator_line }}) || echo failed_to_get_info"
+ register: multipath_info
+ - name: Save multipath info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ multipath_info.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_multipath_info"
+
+ - name: Get info about devices
+ shell: '(echo -e "{{ separator_line }}\nlsblk info:"; lsblk;
+ echo -e "{{ separator_line }}\nPVs info:"; pvs;
+ echo -e "{{ separator_line }}\nVGs info:"; vgs;
+ echo -e "{{ separator_line }}\nLVs info:"; lvs;
+ echo -e "{{ separator_line }}\nll /dev/disk/by-path/ip-*:";
+ ll /dev/disk/by-path/ip-*; echo {{ separator_line }}) ||
+ echo failed_to_get_info'
+ register: lsblk_pvs_vgs_lvs
+ - name: Save devices info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ lsblk_pvs_vgs_lvs.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_lsblk_pvs_vgs_lvs"
+
+ - name: Get info about mounts
+ shell: '(echo -e "Mounts on the {{ current_hostname }} node:\n"; mount) ||
+ echo failed_to_get_info'
+ register: mounts
+ - name: Save mounts info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ mounts.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_mount"
+
+ - name: Get info about space usage
+ shell: '(echo -e "File system disk space usage on the {{
+ current_hostname }} node:\n"; df -Th) ||echo failed_to_get_df_info'
+ register: df_info
+ - name: Save mounts info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ df_info.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_df"
+
+ - name: Read 'dmesg -T' info
+ shell: "dmesg -T || echo failed_to_get_info"
+ register: dmesg_info
+ - name: Save dmesg info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ dmesg_info.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_dmesg"
+
+ - name: Create archive from '/var/log/messages' dir
+ shell: "tar -czvf var_log_messages.tar.gz /var/log/messages"
+ retries: 15
+ delay: 2
+ register: result
+ until: result is succeeded
+ ignore_errors: yes
+ - name: Copy the archive to the localhost
+ fetch:
+ src: "var_log_messages.tar.gz"
+ dest: "{{ dir_path }}{{ current_hostname }}_var_log_messages.tar.gz"
+ flat: yes
+ fail_on_missing: yes
+ ignore_errors: yes
diff --git a/deployment/playbooks/get_ocp_info.yaml b/deployment/playbooks/get_ocp_info.yaml
index dfce216c..7046ccc6 100644
--- a/deployment/playbooks/get_ocp_info.yaml
+++ b/deployment/playbooks/get_ocp_info.yaml
@@ -74,7 +74,9 @@
- glusterd
- heketi
- gluster-blockd
+ - gluster-block-target
- tcmu-runner
+ - rpcbind
heketi_pod_package_list:
- gluster
- heketi
diff --git a/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml b/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml
index c6c8cf27..159eb93e 100644
--- a/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml
+++ b/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml
@@ -15,6 +15,11 @@
when: s.stat.exists
- block:
+ - name: Install 'sos' package
+ yum:
+ name: sos
+ state: installed
+ ignore_errors: yes
- name: be sure all pre-req packages are installed
yum: name={{item}} state=installed
with_items: