summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--deployment/playbooks/gather_logs.yaml883
-rw-r--r--deployment/playbooks/get_ocp_info.yaml2
-rw-r--r--deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml5
3 files changed, 890 insertions, 0 deletions
diff --git a/deployment/playbooks/gather_logs.yaml b/deployment/playbooks/gather_logs.yaml
new file mode 100644
index 00000000..33b9114f
--- /dev/null
+++ b/deployment/playbooks/gather_logs.yaml
@@ -0,0 +1,883 @@
+# Expected vars:
+#
+# - 'config_filepath' - required. It is expected to be provided and must
+# store filepath for the config file used by automated test caes.
+# Current playbook will take all the nodes info from it.
+#
+# - 'output_artifacts_dir' - optional. It is directory where should be saved
+# generated/gathered files.
+#
+# Command to run this playbook:
+#
+# $ tox -e ocp3.11 -- ansible-playbook -i 127.0.0.1, \
+# playbooks/gather_logs.yaml \
+# -e config_filepath=/path/to/the/cns-automation-config.yaml \
+# -e output_artifacts_dir=../cluster_logs/
+
+---
+- hosts: localhost
+ connection: local
+ gather_facts: no
+ tasks:
+ - name: Process config file and find all the nodes of an OpenShift cluster
+ command: "python -c \"import yaml ;
+ config = yaml.load(open('{{ config_filepath }}', 'r'));
+ print(' '.join(list(config['ocp_servers']['master'].keys())));
+ print(' '.join(list(config['ocp_servers']['nodes'].keys())));
+ print(' '.join(list(config['gluster_servers'].keys())));
+ print(config.get('openshift', config.get('cns',
+ {}))['heketi_config']['heketi_server_url']);
+ print(config.get('openshift', config.get('cns',
+ {}))['heketi_config']['heketi_cli_user']);
+ print(config.get('openshift', config.get('cns',
+ {}))['heketi_config']['heketi_cli_key'])\""
+ register: config_data
+ - debug:
+ msg: "{{ config_data }}"
+ - name: Save config data to the host vars
+ set_fact:
+ master_nodes: "{{ config_data.stdout_lines[0].split(' ') }}"
+ compute_nodes: "{{ config_data.stdout_lines[1].split(' ') }}"
+ gluster_nodes: "{{ config_data.stdout_lines[2].split(' ') }}"
+ heketi_server_url: "{{ config_data.stdout_lines[3] }}"
+ heketi_cli_user: "{{ config_data.stdout_lines[4] }}"
+ heketi_cli_key: "{{ config_data.stdout_lines[5] }}"
+ - name: Print list of master nodes IP addresses
+ debug:
+ msg: "{{ master_nodes }}"
+ - name: Print list of compute nodes IP addresses
+ debug:
+ msg: "{{ compute_nodes }}"
+ - name: Print list of gluster nodes IP addresses
+ debug:
+ msg: "{{ gluster_nodes }}"
+ - name: Add gathered master IP addresses to the Ansible host list
+ add_host:
+ hostname: "{{ item }}"
+ ansible_host: "{{ item }}"
+ groups: logs_ocp_nodes, logs_single_master_node, logs_master_nodes
+ with_items: "{{ master_nodes }}"
+ - name: Add gathered compute IP addresses to the Ansible host list
+ add_host:
+ hostname: "{{ item }}"
+ ansible_host: "{{ item }}"
+ groups: logs_ocp_nodes, logs_compute_nodes
+ with_items: "{{ compute_nodes }}"
+ - name: Add gathered gluster IP addresses to the Ansible host list
+ add_host:
+ hostname: "{{ item }}"
+ ansible_host: "{{ item }}"
+ groups: logs_ocp_nodes, logs_gluster_nodes
+ with_items: "{{ gluster_nodes }}"
+
+- hosts: all
+ gather_facts: no
+ tasks:
+ - set_fact:
+ output_artifacts_dir: "{{
+ (output_artifacts_dir | default('../cluster_logs/')) }}"
+ - set_fact:
+ output_artifacts_dir: "{{ output_artifacts_dir
+ }}{% if output_artifacts_dir[-1] != '/' %}{{ '/' }}{% endif %}"
+ - name: Get hostname of the current host
+ shell: "hostname"
+ register: hostname
+ - name: Save hostname to the var
+ set_fact:
+ current_hostname: "{{ hostname.stdout_lines[0].strip() }}"
+ separator_line: "{{ '=' * 79 }}"
+ - name: Create output artifacts directory if absent
+ delegate_to: localhost
+ file:
+ path: "{{ output_artifacts_dir }}"
+ state: directory
+ run_once: yes
+ - name: Install 'rsync' package which is required by 'synchronize' module
+ yum:
+ name: rsync
+ state: installed
+ ignore_errors: yes
+
+# All nodes SOS reports
+- hosts: logs_ocp_nodes
+ gather_facts: no
+ tasks:
+ - name: Install 'sos' package if absent
+ package:
+ name: sos
+ state: present
+ - name: Run sosreport command
+ shell: "sosreport --batch --verbose --tmp-dir . --label customuniquelabel"
+ register: sosreport_output
+ - name: Print the output of the sosreport command
+ debug:
+ msg: "{{ sosreport_output }}"
+ - name: Get name of the generated sos-file
+ shell: 'echo -e "{{ sosreport_output.stdout }}" | grep customuniquelabel'
+ register: sos_filepath
+ - name: Copy generated files to the localhost
+ fetch:
+ src: "{{ sos_filepath.stdout_lines[0].strip() }}"
+ dest: "{{ output_artifacts_dir }}sosreports/"
+ flat: yes
+ fail_on_missing: yes
+
+- hosts: logs_single_master_node
+ gather_facts: no
+ vars:
+ master_package_list:
+ - docker
+ - heketi
+ master_service_list:
+ - docker
+ - multipathd
+ heketi_pod_package_list:
+ - gluster
+ - heketi
+ gluster_pod_package_list:
+ - gluster
+ - heketi
+ - targetcli
+ - gluster-block
+ - tcmu-runner
+ - python-configshell
+ - python-rtslib
+ gluster_pod_service_list:
+ - glusterd
+ - heketi
+ - gluster-blockd
+ - gluster-block-target
+ - tcmu-runner
+ - rpcbind
+ tasks:
+
+ # Master node info
+ - name: Get distro version
+ shell: "uname -a"
+ register: master_linux_kernel_version
+ - name: Get Red Hat release info
+ shell: "cat /etc/redhat-release"
+ register: master_rh_release
+ - name: Create grep filter with all the packages we are interested in
+ set_fact:
+ package_filter: "{{ package_filter | default('grep') + ' -e ' + item }}"
+ with_items: "{{ master_package_list }}"
+ - name: Get list of installed packages we are interested in
+ shell: "rpm -qa | {{ package_filter }}"
+ register: master_packages
+ - name: Get status of services on OCP Master node
+ shell: "systemctl list-units {{ master_service_list | join('.service ') }}.service
+ --type=service --all --no-pager --no-legend"
+ register: master_services
+ - name: Get OpenShift client version
+ shell: "(oc version | grep -e 'oc ' -e 'openshift' -e 'kube') ||
+ echo failed_to_get_oc_version_info"
+ register: master_oc_version
+ - name: Get list of OCP nodes
+ shell: "oc get nodes -o wide || echo failed_to_get_list_of_nodes"
+ register: master_ocp_nodes
+ - name: Get info about all the docker images used in OCP cluster
+ shell: "(oc get pods --all-namespaces
+ -o=custom-columns=:.status.containerStatuses[*].image | grep -v -e '^$' | uniq) ||
+ echo failed_to_get_list_of_images"
+ register: master_image_info
+ - name: Write master data to the data file
+ delegate_to: localhost
+ yedit:
+ src: "{{ output_artifacts_dir }}master_data.yaml"
+ state: present
+ edits:
+ - key: master
+ value:
+ Linux kernel version: "{{ master_linux_kernel_version.stdout_lines }}"
+ Red Hat release info: "{{ master_rh_release.stdout_lines }}"
+ List of Packages: "{{ master_packages.stdout_lines }}"
+ List of services: "{{ master_services.stdout_lines }}"
+ OC Version: "{{ master_oc_version.stdout_lines }}"
+ OCP nodes: "{{ master_ocp_nodes.stdout_lines }}"
+ Images info: "{{ master_image_info.stdout_lines }}"
+
+ # Heketi POD logs, config and DB dump
+ - name: Get heketi POD
+ shell: "oc get pods --all-namespaces -l heketi --no-headers
+ -o=custom-columns=:.metadata.name,:.metadata.namespace"
+ register: heketi_pods
+ retries: 10
+ delay: 6
+ until: heketi_pods is succeeded
+ - name: DEBUG HEKETI
+ debug:
+ msg: "{{ heketi_pods }}"
+ - block:
+ - name: Create var with destination dir path
+ set_fact:
+ dir_path: "{{ output_artifacts_dir }}heketi_pod/"
+ - name: Create compute directory if absent
+ delegate_to: localhost
+ file:
+ path: "{{ dir_path }}"
+ state: directory
+ - name: Set Heketi POD name and Heketi namespace as vars
+ set_fact:
+ heketi_pod_name: "{{
+ (heketi_pods.stdout_lines[0].split(' ') | list)[0] }}"
+ heketi_pod_namespace: "{{
+ (heketi_pods.stdout_lines[0].split(' ') | list)[-1] }}"
+ - name: Set Heketi pod command prefix
+ set_fact:
+ heketi_pod_cmd_prefix: "oc exec {{ heketi_pod_name
+ }} --namespace {{ heketi_pod_namespace }} --"
+ - name: Get the Heketi config from the Heketi POD
+ shell: 'echo -e "$({{ heketi_pod_cmd_prefix
+ }} cat /etc/heketi/heketi.json ||
+ echo error_failed_to_get_the_heketi_config_file)" |
+ tee /tmp/heketi_config.json'
+ - name: Copy the Heketi config
+ fetch:
+ src: "/tmp/heketi_config.json"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+ - name: Save Heketi POD logs
+ shell: "(oc logs {{ heketi_pod_name }} ||
+ echo 'ERROR! Failed to get the Heketi logs.') > /tmp/heketi.log"
+ - name: Copy the Heketi logs
+ fetch:
+ src: "/tmp/heketi.log"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+ - name: Dump the Heketi DB
+ shell: 'echo -e "$({{ heketi_pod_cmd_prefix }} heketi-cli --server {{
+ hostvars["localhost"]["heketi_server_url"] }} --user {{
+ hostvars["localhost"]["heketi_cli_user"] }} --secret {{
+ hostvars["localhost"]["heketi_cli_key"]
+ }} db dump ||
+ echo \{\"error\"\:\"failed_to_get_the_Heketi_db_dump\"\})" |
+ python -m json.tool > /tmp/heketi_db_dump.json'
+ - name: Copy the Heketi DB dump
+ fetch:
+ src: "/tmp/heketi_db_dump.json"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+ - name: Get storage release version from Heketi POD
+ shell: "{{ heketi_pod_cmd_prefix }} cat /etc/redhat-storage-release ||
+ echo failed_to_read_redhat_storage_release_info"
+ register: heketi_pod_storage_release_version
+ - name: Get info about packages on Heketi POD
+ shell: "{{ heketi_pod_cmd_prefix }} rpm -qa | grep -e {{
+ heketi_pod_package_list | join(' -e ') }} ||
+ echo failed_to_read_list_of_installed_packages"
+ register: heketi_pod_packages
+ - name: Write Heketi data to the data file
+ delegate_to: localhost
+ yedit:
+ src: "{{ dir_path }}heketi_pod_data.yaml"
+ state: present
+ edits:
+ - key: "Storage_release_version"
+ value: "{{ heketi_pod_storage_release_version.stdout_lines }}"
+ - key: "List_of_Packages"
+ value: "{{ heketi_pod_packages.stdout_lines }}"
+ when: "((heketi_pods.stdout_lines | join('')).strip() | length) > 0"
+
+ # Gluster PODs
+ - name: Get list of Gluster PODs
+ shell: "oc get pods --all-namespaces -l glusterfs-node --no-headers
+ -o=custom-columns=:.metadata.name,:.metadata.namespace"
+ register: gluster_pods
+ retries: 10
+ delay: 6
+ until: gluster_pods is succeeded
+ - name: DEBUG GLUSTER
+ debug:
+ msg: "{{ gluster_pods }}"
+ - name: Create var describing the Gluster cluster deployment type
+ set_fact:
+ is_gluster_containerized: "{{
+ ((gluster_pods.stdout_lines | join('')).strip() | length) > 0 }}"
+ delegate_to: "{{ item }}"
+ delegate_facts: yes
+ with_items: "{{ groups['all'] }}"
+ - block:
+ - name: Create var with destination dir path
+ set_fact:
+ dir_path: "{{ output_artifacts_dir }}gluster_pods/"
+ - name: Create directory if absent
+ delegate_to: localhost
+ file:
+ path: "{{ dir_path }}"
+ state: directory
+ - name: Define storage namespace
+ set_fact:
+ storage_namespace: "{{ (gluster_pods.stdout_lines[0].split(
+ ' ') | list)[-1] }}"
+ - name: Define gluster POD names
+ set_fact:
+ gluster_pod_names: "{{ (gluster_pod_names | default([])) +
+ [(item.split(' ') | list)[0]] }}"
+ with_items: "{{ gluster_pods.stdout_lines[0:] }}"
+ - debug:
+ msg: "{{ gluster_pod_names }}"
+
+ - name: Get storage release version from Gluster PODs
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} --
+ cat /etc/redhat-storage-release) ||
+ echo failed_to_get_redhat_storage_release_info"
+ with_items: "{{ gluster_pod_names }}"
+ register: gluster_pod_storage_release_version_results
+ - name: Process gluster PODs storage release versions results
+ set_fact:
+ gluster_pod_storage_release_version_processed: "{{
+ gluster_pod_storage_release_version_processed | default({}) | combine(
+ {(item.item.strip().split(' ')[0]): item.stdout_lines},
+ recursive=True
+ ) }}"
+ with_items: "{{ gluster_pod_storage_release_version_results.results }}"
+ - name: Get info about packages on Gluster PODs
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} --
+ rpm -qa | grep -e {{ gluster_pod_package_list | join(' -e ') }}) ||
+ echo failed_to_get_packages_info_from_gluster_pod"
+ with_items: "{{ gluster_pod_names }}"
+ register: gluster_pod_package_list_results
+ - name: Process gluster PODs package lists results
+ set_fact:
+ gluster_pod_package_list_processed: "{{
+ gluster_pod_package_list_processed | default({}) | combine(
+ {(item.item.strip().split(' ')[0]): item.stdout_lines},
+ recursive=True
+ ) }}"
+ with_items: "{{ gluster_pod_package_list_results.results }}"
+ - name: Get info about services on Gluster PODs
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} --
+ systemctl list-units {{
+ gluster_pod_service_list | join('.service ') }}.service
+ --type=service --all --no-pager --no-legend) ||
+ echo failed_to_get_services_info_from_gluster_pod"
+ with_items: "{{ gluster_pod_names }}"
+ register: gluster_pod_service_list_results
+ - name: Process gluster PODs service lists results
+ set_fact:
+ gluster_pod_service_list_processed: "{{
+ gluster_pod_service_list_processed | default({}) | combine(
+ {(item.item.strip().split(' ')[0]): item.stdout_lines},
+ recursive=True
+ ) }}"
+ with_items: "{{ gluster_pod_service_list_results.results }}"
+ - name: Write Gluster PODs data to the data file
+ delegate_to: localhost
+ yedit:
+ src: "{{ dir_path }}gluster_pods_packages_and_services_data.yaml"
+ state: present
+ edits:
+ - key: gluster_pods
+ value:
+ Storage release version: "{{
+ gluster_pod_storage_release_version_processed }}"
+ List of Packages: "{{ gluster_pod_package_list_processed }}"
+ List of Services: "{{ gluster_pod_service_list_processed }}"
+
+ - name: Get 'targetcli ls' output
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} --
+ targetcli ls) || echo failed_to_get_targetcli_ls_output"
+ with_items: "{{ gluster_pod_names }}"
+ register: gluster_pod_targetcli_ls_results
+ - debug:
+ msg: "{{ gluster_pod_targetcli_ls_results }}"
+ - name: Write Gluster PODs data to the data file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-targetcli-ls"
+ with_items: "{{ gluster_pod_targetcli_ls_results.results }}"
+
+ - name: Create gluster log directories on the master node
+ file:
+ path: "/tmp/gluster/{{ item }}-var_log_glusterfs"
+ state: directory
+ with_items: "{{ gluster_pod_names }}"
+ - name: Copy '/var/log/glusterfs/*' files to the master
+ shell: "(oc cp {{ storage_namespace }}/{{ item }}:/var/log/glusterfs
+ /tmp/gluster/{{ item }}-var_log_glusterfs) ||
+ echo failed_to_copy_var_log_glusterfs_files"
+ with_items: "{{ gluster_pod_names }}"
+ - name: Copy '/var/log/glusterfs/*' files from the master to the localhost
+ synchronize:
+ src: "/tmp/gluster/{{ item }}-var_log_glusterfs/"
+ dest: "{{ dir_path }}{{ item }}-var_log_glusterfs/"
+ mode: pull
+ recursive: yes
+ use_ssh_args: yes
+ with_items: "{{ gluster_pod_names }}"
+ ignore_errors: yes
+
+ - name: Get 'dmesg -T' info
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- dmesg -T) || echo failed_toget_dmesg_-T_info"
+ with_items: "{{ gluster_pod_names }}"
+ register: gluster_pods_dmesg_results
+ - name: Write Gluster PODs dmesg data to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-dmesg"
+ with_items: "{{ gluster_pods_dmesg_results.results }}"
+
+ - name: Get list of processes and info for processes with 'D' stat
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace }} -- ps aux ;
+ oc exec {{ item }} --namespace {{ storage_namespace }} --
+ ps -aux | tee /dev/tty | awk {'if ( $8 ~ \"D\" ) print $2'} |
+ while read -r pid ;
+ do echo -e \"\nRunning '/proc/$pid/stack' command:\";
+ cat /proc/$pid/stack ;
+ done) || echo failed_to_get_info_about_processes_with_D_stat"
+ with_items: "{{ gluster_pod_names }}"
+ register: gluster_pod_io_processes_info
+ - name: Write Gluster PODs 'I\O' proccesses info to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-ps"
+ with_items: "{{ gluster_pod_io_processes_info.results }}"
+
+ - name: List dirs and files in '/sys/module/dm_multipath'
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- ls -l /sys/module/dm_multipath) ||
+ echo failed_to_list_files_in__sys_module_dm_multipath"
+ with_items: "{{ gluster_pod_names }}"
+ register: sys_module_dm_multipath_results
+ - name: Write Gluster PODs 'ls -l /sys/module/dm_multipath' output to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0]
+ }}-ls-sys_module_dm_multipath"
+ with_items: "{{ sys_module_dm_multipath_results.results }}"
+
+ - name: "Run 'lsmod | egrep target_core' command"
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- lsmod | egrep target_core) ||
+ echo failed_to_get_lsmod_info_for_target_core"
+ with_items: "{{ gluster_pod_names }}"
+ register: lsmod_target_core_results
+ - name: Write Gluster PODs 'lsmod | egrep target_core' command results to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{
+ (item.item.strip().split(' ') | list)[0] }}-lsmod_target_core"
+ with_items: "{{ lsmod_target_core_results.results }}"
+
+ - name: Get info about devices
+ shell: '(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- bash -c "echo -e \"{{ separator_line }}\nlsblk info:\"; lsblk;
+ echo -e \"{{ separator_line }}\nPVs info:\"; pvs;
+ echo -e \"{{ separator_line }}\nVGs info:\"; vgs;
+ echo -e \"{{ separator_line }}\nLVs info:\"; lvs;
+ echo -e \"{{ separator_line }}\"") ||
+ echo failed_to_get_list_of_pvs_vgs_and_lvs'
+ with_items: "{{ gluster_pod_names }}"
+ register: lsblk_pvs_vgs_lvs_results
+ - name: Write Gluster PODs info about PVs, VGs and LVs to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0]
+ }}-lsblk_pvs_vgs_lvs"
+ with_items: "{{ lsblk_pvs_vgs_lvs_results.results }}"
+
+ - name: Read 'journalctl' output
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- journalctl) || echo failed_to_read_journalctl_output"
+ with_items: "{{ gluster_pod_names }}"
+ register: journalctl_results
+ - name: Write Gluster PODs 'journalctl' output to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0]
+ }}-journalctl"
+ with_items: "{{ journalctl_results.results }}"
+
+ - name: Read 'mount' output
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- mount) || echo failed_to_read_mount_output"
+ with_items: "{{ gluster_pod_names }}"
+ register: mount_results
+ - name: Write Gluster PODs 'mount' output to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ item.stdout }}"
+ dest: "{{ dir_path }}{{ (item.item.strip().split(' ') | list)[0] }}-mount"
+ with_items: "{{ mount_results.results }}"
+
+ - name: Create archive from ' /etc/target/' dir
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- tar -czvf etc_target.tar.gz /etc/target/ ) ||
+ echo failed_to_archive__etc_target_dir"
+ with_items: "{{ gluster_pod_names }}"
+ - name: Copy archive of the '/etc/target/' dir to the master
+ shell: "(oc cp {{ storage_namespace }}/{{ item }}:/etc_target.tar.gz
+ /tmp/gluster/{{ item }}-etc_target.tar.gz) ||
+ echo failed_to_copy_etc_target_file"
+ with_items: "{{ gluster_pod_names }}"
+ - name: Copy archive of the '/etc/target/' dir to the localhost
+ fetch:
+ src: "/tmp/gluster/{{ item }}-etc_target.tar.gz"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+ with_items: "{{ gluster_pod_names }}"
+
+ - name: Create archive from '/sys/kernel/config/target/' dir
+ shell: "(oc exec {{ item }} --namespace {{ storage_namespace
+ }} -- tar -czvf sys_kernel_config_target.tar.gz /sys/kernel/config/target/ ) ||
+ echo failed_to_archive__sys_kernel_config_target_dir"
+ with_items: "{{ gluster_pod_names }}"
+ - name: Copy archive of the '/sys/kernel/config/target/' dir to the master
+ shell: "(oc cp {{ storage_namespace }}/{{ item }}:/sys_kernel_config_target.tar.gz
+ /tmp/gluster/{{ item }}-sys_kernel_config_target.tar.gz) ||
+ echo failed_to_copy_sys_kernel_config_target_file"
+ with_items: "{{ gluster_pod_names }}"
+ - name: Copy archive of the '/sys/kernel/config/target/' dir to the localhost
+ fetch:
+ src: "/tmp/gluster/{{ item }}-sys_kernel_config_target.tar.gz"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+ with_items: "{{ gluster_pod_names }}"
+ when: "is_gluster_containerized"
+
+# Gather info from gluster nodes in case of 'standalone' deployment
+- hosts: logs_gluster_nodes
+ gather_facts: no
+ vars:
+ gluster_package_list:
+ - gluster
+ - heketi
+ - targetcli
+ - gluster-block
+ - tcmu-runner
+ - python-configshell
+ - python-rtslib
+ gluster_service_list:
+ - glusterd
+ - heketi
+ - gluster-blockd
+ - gluster-block-target
+ - tcmu-runner
+ - rpcbind
+ tasks:
+ - block:
+ - name: Create var with destination dir path
+ set_fact:
+ dir_path: "{{ output_artifacts_dir }}gluster_nodes/"
+ - name: Create directory if absent
+ delegate_to: localhost
+ file:
+ path: "{{ dir_path }}"
+ state: directory
+ run_once: yes
+
+ - name: Get storage release version from Gluster node
+ shell: "(cat /etc/redhat-storage-release) ||
+ echo failed_to_get_redhat_storage_release_info"
+ register: gluster_node_storage_release_version_results
+ - name: Get info about packages on Gluster node
+ shell: "(rpm -qa | grep -e {{ gluster_package_list | join(' -e ') }}) ||
+ echo failed_to_get_packages_info_from_gluster_node"
+ register: gluster_node_package_list_results
+ - name: Get info about services on Gluster node
+ shell: "(systemctl list-units {{
+ gluster_service_list | join('.service ') }}.service
+ --type=service --all --no-pager --no-legend) ||
+ echo failed_to_get_services_info_from_gluster_node"
+ register: gluster_node_service_list_results
+ - name: Write Gluster node data to the data file
+ delegate_to: localhost
+ yedit:
+ src: "{{ dir_path }}gluster_nodes_packages_and_services_data.yaml"
+ state: present
+ edits:
+ - key: "gluster_node_{{ current_hostname }}"
+ value:
+ Storage release version: "{{
+ gluster_node_storage_release_version_results.stdout }}"
+ List of Packages: "{{ gluster_node_package_list_results.stdout_lines }}"
+ List of Services: "{{ gluster_node_service_list_results.stdout_lines }}"
+
+ - name: Get 'targetcli ls' output
+ shell: "targetcli ls || echo failed_to_get_targetcli_ls_output"
+ register: gluster_node_targetcli_ls_results
+ - name: Write Gluster node data to the data file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ gluster_node_targetcli_ls_results.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-targetcli-ls"
+
+ - name: Copy '/var/log/glusterfs/*' files from the current gluster node
+ synchronize:
+ src: "/var/log/glusterfs/"
+ dest: "{{ dir_path }}{{ current_hostname }}-var_log_glusterfs/"
+ mode: pull
+ recursive: yes
+ use_ssh_args: yes
+ ignore_errors: yes
+
+ - name: Get info about space usage
+ shell: '(echo -e "File system disk space usage on the {{
+ current_hostname }} node:\n"; df -Th) ||echo failed_to_get_df_info'
+ register: df_info
+ - name: Save mounts info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ df_info.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_df"
+
+ - name: Get 'dmesg -T' info
+ shell: "dmesg -T || echo failed_toget_dmesg_-T_info"
+ register: gluster_node_dmesg_results
+ - name: Write Gluster node dmesg data to files
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ gluster_node_dmesg_results.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-dmesg"
+
+ - name: Get list of processes and info for processes with 'D' stat
+ shell: "( ps aux ;
+ ps -aux | tee /dev/tty | awk {'if ( $8 ~ \"D\" ) print $2'} |
+ while read -r pid ;
+ do echo -e \"\nRunning '/proc/$pid/stack' command:\";
+ cat /proc/$pid/stack ;
+ done) || echo failed_to_get_info_about_processes_with_D_stat"
+ register: gluster_node_io_processes_info
+ - name: Write Gluster node 'I\O' proccesses info to a file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ gluster_node_io_processes_info.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-ps"
+
+ - name: List dirs and files in '/sys/module/dm_multipath'
+ shell: "ls -l /sys/module/dm_multipath ||
+ echo failed_to_list_files_in__sys_module_dm_multipath"
+ register: sys_module_dm_multipath_results
+ - name: Write Gluster node 'ls -l /sys/module/dm_multipath' output to a file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ sys_module_dm_multipath_results.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-ls-sys_module_dm_multipath"
+
+ - name: "Run 'lsmod | egrep target_core' command"
+ shell: "(lsmod | egrep target_core) ||
+ echo failed_to_get_lsmod_info_for_target_core"
+ register: lsmod_target_core_results
+ - name: Write Gluster node 'lsmod | egrep target_core' command results to a file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ lsmod_target_core_results.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-lsmod_target_core"
+
+ - name: Get info about devices
+ shell: '(echo -e "{{ separator_line }}\nlsblk info:"; lsblk;
+ echo -e "{{ separator_line }}\nPVs info:"; pvs;
+ echo -e "{{ separator_line }}\nVGs info:"; vgs;
+ echo -e "{{ separator_line }}\nLVs info:"; lvs;
+ echo -e "{{ separator_line }}\nll /dev/disk/by-path/ip-*:";
+ ll /dev/disk/by-path/ip-*; echo {{ separator_line }}) ||
+ echo failed_to_get_info'
+ register: lsblk_pvs_vgs_lvs
+ - name: Save devices info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ lsblk_pvs_vgs_lvs.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_lsblk_pvs_vgs_lvs"
+
+ - name: Read 'journalctl' output
+ shell: "journalctl || echo failed_to_read_journalctl_output"
+ register: journalctl_results
+ - name: Write Gluster node 'journalctl' output to a file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ journalctl_results.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-journalctl"
+
+ - name: Read 'mount' output
+ shell: "mount || echo failed_to_read_mount_output"
+ register: mount_results
+ - name: Write Gluster node 'mount' output to a file
+ delegate_to: localhost
+ no_log: yes
+ copy:
+ content: "{{ mount_results.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}-mount"
+
+ - name: Create archive from ' /etc/target/' dir
+ shell: "tar -czvf /tmp/gluster/{{ current_hostname
+ }}-etc_target.tar.gz /etc/target/ ||
+ echo failed_to_archive__etc_target_dir"
+ - name: Copy archive of the '/etc/target/' dir to the localhost
+ fetch:
+ src: "/tmp/gluster/{{ current_hostname }}-etc_target.tar.gz"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+
+ - name: Create archive from '/sys/kernel/config/target/' dir
+ shell: "tar -czvf /tmp/gluster/{{ current_hostname
+ }}-sys_kernel_config_target.tar.gz /sys/kernel/config/target/ ||
+ echo failed_to_archive__sys_kernel_config_target_dir"
+ - name: Copy archive of the '/sys/kernel/config/target/' dir to the localhost
+ fetch:
+ src: "/tmp/gluster/{{ current_hostname }}-sys_kernel_config_target.tar.gz"
+ dest: "{{ dir_path }}"
+ flat: yes
+ fail_on_missing: yes
+
+ - name: Create archive from '/var/log/messages' dir
+ shell: "tar -czvf var_log_messages.tar.gz /var/log/messages"
+ retries: 15
+ delay: 2
+ register: result
+ until: result is succeeded
+ ignore_errors: yes
+ - name: Copy the archive to the localhost
+ fetch:
+ src: "var_log_messages.tar.gz"
+ dest: "{{ dir_path }}{{ current_hostname }}_var_log_messages.tar.gz"
+ flat: yes
+ fail_on_missing: yes
+ ignore_errors: yes
+ when: "not is_gluster_containerized"
+
+# Gather info from compute nodes
+- hosts: logs_compute_nodes
+ gather_facts: no
+ vars:
+ compute_package_list:
+ - docker
+ - heketi
+ - rpcbind
+ compute_service_list:
+ - docker
+ - multipathd
+ - rpcbind
+ - iscsid
+ tasks:
+ - name: Create var with destination dir path
+ set_fact:
+ dir_path: "{{ output_artifacts_dir }}compute_nodes/"
+ - name: Create compute directory if absent
+ delegate_to: localhost
+ file:
+ path: "{{ dir_path }}"
+ state: directory
+ run_once: yes
+
+ - name: Create grep filter with all the packages we are interested in
+ set_fact:
+ package_filter: "{{ package_filter | default('grep') + ' -e ' + item }}"
+ with_items: "{{ compute_package_list }}"
+ - name: Get list of installed packages we are interested in
+ shell: "rpm -qa | {{ package_filter }} || echo failed_to_get_info"
+ register: compute_packages
+ - name: Write compute nodes package list to a file
+ delegate_to: localhost
+ copy:
+ content: "{{ compute_packages.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_packages.yaml"
+
+ - name: Get status of services on OCP Compute nodes
+ shell: "systemctl list-units {{
+ compute_service_list | join('.service ') }}.service --no-pager
+ --type=service --all --no-legend || echo failed_to_get_info"
+ register: compute_services
+ - name: Write compute nodes service list to a file
+ delegate_to: localhost
+ copy:
+ content: "{{ compute_services.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_services.yaml"
+
+ - name: Get multipath info
+ shell: "(echo 'Multipath config is following:'; cat /etc/multipath.conf;
+ echo -e \"{{ separator_line }}\nResults of 'nmultipath -ll' command:\";
+ multipath -ll; echo {{ separator_line }}) || echo failed_to_get_info"
+ register: multipath_info
+ - name: Save multipath info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ multipath_info.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_multipath_info"
+
+ - name: Get info about devices
+ shell: '(echo -e "{{ separator_line }}\nlsblk info:"; lsblk;
+ echo -e "{{ separator_line }}\nPVs info:"; pvs;
+ echo -e "{{ separator_line }}\nVGs info:"; vgs;
+ echo -e "{{ separator_line }}\nLVs info:"; lvs;
+ echo -e "{{ separator_line }}\nll /dev/disk/by-path/ip-*:";
+ ll /dev/disk/by-path/ip-*; echo {{ separator_line }}) ||
+ echo failed_to_get_info'
+ register: lsblk_pvs_vgs_lvs
+ - name: Save devices info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ lsblk_pvs_vgs_lvs.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_lsblk_pvs_vgs_lvs"
+
+ - name: Get info about mounts
+ shell: '(echo -e "Mounts on the {{ current_hostname }} node:\n"; mount) ||
+ echo failed_to_get_info'
+ register: mounts
+ - name: Save mounts info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ mounts.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_mount"
+
+ - name: Get info about space usage
+ shell: '(echo -e "File system disk space usage on the {{
+ current_hostname }} node:\n"; df -Th) ||echo failed_to_get_df_info'
+ register: df_info
+ - name: Save mounts info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ df_info.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_df"
+
+ - name: Read 'dmesg -T' info
+ shell: "dmesg -T || echo failed_to_get_info"
+ register: dmesg_info
+ - name: Save dmesg info into a file
+ delegate_to: localhost
+ copy:
+ content: "{{ dmesg_info.stdout }}"
+ dest: "{{ dir_path }}{{ current_hostname }}_dmesg"
+
+ - name: Create archive from '/var/log/messages' dir
+ shell: "tar -czvf var_log_messages.tar.gz /var/log/messages"
+ retries: 15
+ delay: 2
+ register: result
+ until: result is succeeded
+ ignore_errors: yes
+ - name: Copy the archive to the localhost
+ fetch:
+ src: "var_log_messages.tar.gz"
+ dest: "{{ dir_path }}{{ current_hostname }}_var_log_messages.tar.gz"
+ flat: yes
+ fail_on_missing: yes
+ ignore_errors: yes
diff --git a/deployment/playbooks/get_ocp_info.yaml b/deployment/playbooks/get_ocp_info.yaml
index dfce216c..7046ccc6 100644
--- a/deployment/playbooks/get_ocp_info.yaml
+++ b/deployment/playbooks/get_ocp_info.yaml
@@ -74,7 +74,9 @@
- glusterd
- heketi
- gluster-blockd
+ - gluster-block-target
- tcmu-runner
+ - rpcbind
heketi_pod_package_list:
- gluster
- heketi
diff --git a/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml b/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml
index c6c8cf27..159eb93e 100644
--- a/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml
+++ b/deployment/playbooks/roles/vmware-guest-setup/tasks/main.yaml
@@ -15,6 +15,11 @@
when: s.stat.exists
- block:
+ - name: Install 'sos' package
+ yum:
+ name: sos
+ state: installed
+ ignore_errors: yes
- name: be sure all pre-req packages are installed
yum: name={{item}} state=installed
with_items: