Hello,
I run Gitlab-CE in Docker on an Ubuntu 18.04 LTS Server.
The last working Gitlab version I got was 11.11.0. Recently, this server got an apt-upgrade (and the latest docker version) and with this, I tried installing the newest Gitlab container following this tutorial: https://docs.gitlab.com/omnibus/docker/#upgrade-gitlab-to-newer-version
Please notice that while installing and updating, I pretty much left the default configuration as-is and only changed the URL and did some Port redirect.
After a restart of the whole system, Gitlab will not start properly anymore. While trying to check system health with ‘gitlab-ctl status’ after executing ‘docker exec -it gitlab /bin/bash’ I noticed that I get kicked out of the container after some time.
I could manage to trace at least one error with ‘docker logs -f gitlab’ which showed:
================================================================================
Error executing action `run` on resource 'bash[migrate gitlab-rails database]'
================================================================================
Mixlib::ShellOut::ShellCommandFailed
------------------------------------
Expected process to exit with [0], but received '1'
---- Begin output of "bash" "/tmp/chef-script20190629-27-1084s7q" ----
STDOUT: == 20190402150158 BackportEnterpriseSchema: migrating =========================
-- column_exists?(:epics, :parent_id)
-> 0.0020s
-- column_exists?(:geo_event_log, :cache_invalidation_event_id)
-> 0.0014s
-- column_exists?(:vulnerability_feedback, :merge_request_id)
-> 0.0011s
-- column_exists?(:appearances, :updated_by)
-> 0.0013s
-- change_column_null(:application_settings, :password_authentication_enabled_for_git, false)
-> 0.0014s
-- change_table(:application_settings, {})
-> 0.4129s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:application_settings, :custom_project_templates_group_id, {:algorithm=>:concurrently})
-> 0.0036s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:application_settings, :file_template_project_id, {:algorithm=>:concurrently})
-> 0.0017s
-- column_exists?(:boards, :milestone_id)
-> 0.0012s
-- column_exists?(:boards, :weight)
-> 0.0010s
-- column_exists?(:boards, :name)
-> 0.0010s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:boards, :milestone_id, {:algorithm=>:concurrently})
-> 0.0018s
-- column_exists?(:clusters_applications_prometheus, :encrypted_alert_manager_token)
-> 0.0012s
-- column_exists?(:clusters_applications_prometheus, :encrypted_alert_manager_token_iv)
-> 0.0012s
-- column_exists?(:clusters_applications_prometheus, :last_update_started_at)
-> 0.0010s
-- column_exists?(:identities, :saml_provider_id)
-> 0.0010s
-- column_exists?(:identities, :secondary_extern_uid)
-> 0.0009s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:identities, :saml_provider_id, {:where=>"saml_provider_id IS NOT NULL", :algorithm=>:concurrently})
-> 0.0014s
-- column_exists?(:issues, :service_desk_reply_to)
-> 0.0025s
-- column_exists?(:issues, :weight)
-> 0.0017s
-- column_exists?(:lists, :milestone_id)
-> 0.0011s
-- column_exists?(:lists, :user_id)
-> 0.0010s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:lists, :milestone_id, {:algorithm=>:concurrently})
-> 0.0022s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:lists, :user_id, {:algorithm=>:concurrently})
-> 0.0022s
-- column_exists?(:members, :ldap)
-> 0.0013s
-- column_exists?(:members, :override)
-> 0.0012s
-- column_exists?(:merge_requests, :approvals_before_merge)
-> 0.0022s
-- column_exists?(:notes, :review_id)
-> 0.0018s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:notes, :review_id, {:algorithm=>:concurrently})
-> 0.0038s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:ci_builds, [:name], {:name=>"index_ci_builds_on_name_for_security_products_values", :where=>"\n (\n (name)::text = ANY (\n ARRAY[\n ('container_scanning'::character varying)::text,\n ('dast'::character varying)::text,\n ('dependency_scanning'::character varying)::text,\n ('license_management'::character varying)::text,\n ('sast'::character varying)::text\n ]\n )\n )", :algorithm=>:concurrently})
-> 0.0070s
-- index_exists?(:environments, :name, {:name=>"index_environments_on_name_varchar_pattern_ops"})
-> 0.0022s
-- change_table(:namespaces, {})
-> 0.0326s
-- column_exists?(:namespaces, :ldap_sync_status)
-> 0.0025s
-- column_exists?(:namespaces, :membership_lock)
-> 0.0019s
-- indexes(:namespaces)
-> 0.0064s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:namespaces, :file_template_project_id, {:algorithm=>:concurrently})
-> 0.0076s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:namespaces, :ldap_sync_last_successful_update_at, {:algorithm=>:concurrently})
-> 0.0054s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:namespaces, :ldap_sync_last_update_at, {:algorithm=>:concurrently})
-> 0.0054s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:namespaces, :plan_id, {:algorithm=>:concurrently})
-> 0.0053s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:namespaces, :trial_ends_on, {:where=>"(trial_ends_on IS NOT NULL)", :algorithm=>:concurrently})
-> 0.0060s
-- indexes(:namespaces)
-> 0.0058s
-- column_exists?(:notification_settings, :new_epic)
-> 0.0019s
-- change_table(:project_mirror_data, {})
-> 0.0052s
-- column_exists?(:project_mirror_data, :retry_count)
-> 0.0010s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:project_mirror_data, :last_successful_update_at, {:algorithm=>:concurrently})
-> 0.0021s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:project_mirror_data, [:next_execution_timestamp, :retry_count], {:name=>"index_mirror_data_on_next_execution_and_retry_count", :algorithm=>:concurrently})
-> 0.0019s
-- column_exists?(:project_statistics, :shared_runners_seconds)
-> 0.0011s
-- column_exists?(:project_statistics, :shared_runners_seconds_last_reset)
-> 0.0010s
-- change_table(:projects, {})
-> 0.0802s
-- change_column_null(:projects, :merge_requests_rebase_enabled, true)
-> 0.0007s
-- column_exists?(:projects, :mirror)
-> 0.0036s
-- column_exists?(:projects, :mirror_trigger_builds)
-> 0.0031s
-- column_exists?(:projects, :reset_approvals_on_push)
-> 0.0039s
-- column_exists?(:projects, :service_desk_enabled)
-> 0.0031s
-- column_exists?(:projects, :approvals_before_merge)
-> 0.0031s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:projects, [:archived, :pending_delete, :merge_requests_require_code_owner_approval], {:name=>"projects_requiring_code_owner_approval", :where=>"((pending_delete = false) AND (archived = false) AND (merge_requests_require_code_owner_approval = true))", :algorithm=>:concurrently})
-> 0.0085s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:projects, [:id, :repository_storage, :last_repository_updated_at], {:name=>"idx_projects_on_repository_storage_last_repository_updated_at", :algorithm=>:concurrently})
-> 0.0079s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:projects, :id, {:name=>"index_projects_on_mirror_and_mirror_trigger_builds_both_true", :where=>"((mirror IS TRUE) AND (mirror_trigger_builds IS TRUE))", :algorithm=>:concurrently})
-> 0.0077s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:projects, :mirror_last_successful_update_at, {:algorithm=>:concurrently})
-> 0.0075s
-- change_column_null(:protected_branch_merge_access_levels, :access_level, true)
-> 0.0004s
-- column_exists?(:protected_branch_merge_access_levels, :group_id)
-> 0.0011s
-- column_exists?(:protected_branch_merge_access_levels, :user_id)
-> 0.0009s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:protected_branch_merge_access_levels, :group_id, {:algorithm=>:concurrently})
-> 0.0014s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:protected_branch_merge_access_levels, :user_id, {:algorithm=>:concurrently})
-> 0.0014s
-- change_column_null(:protected_branch_push_access_levels, :access_level, true)
-> 0.0003s
-- column_exists?(:protected_branch_push_access_levels, :group_id)
-> 0.0009s
-- column_exists?(:protected_branch_push_access_levels, :user_id)
-> 0.0009s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:protected_branch_push_access_levels, :group_id, {:algorithm=>:concurrently})
-> 0.0015s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:protected_branch_push_access_levels, :user_id, {:algorithm=>:concurrently})
-> 0.0013s
-- column_exists?(:resource_label_events, :epic_id)
-> 0.0012s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:resource_label_events, :epic_id, {:algorithm=>:concurrently})
-> 0.0019s
-- column_exists?(:user_preferences, :epic_notes_filter)
-> 0.0012s
-- column_exists?(:user_preferences, :epics_sort)
-> 0.0011s
-- column_exists?(:user_preferences, :roadmap_epics_state)
-> 0.0011s
-- column_exists?(:user_preferences, :roadmaps_sort)
-> 0.0011s
-- column_exists?(:users, :auditor)
-> 0.0034s
-- change_table(:users, {})
-> 0.0376s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:users, :group_view, {:algorithm=>:concurrently})
-> 0.0071s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:users, :managing_group_id, {:algorithm=>:concurrently})
-> 0.0067srake aborted!
StandardError: An error has occurred, all later migrations canceled:
PG::IndexCorrupted: ERROR: index "pg_trigger_tgconstraint_index" contains unexpected zero page at block 0
HINT: Please REINDEX it.
: ALTER TABLE application_settings
ADD CONSTRAINT fk_rails_b53e481273
FOREIGN KEY (custom_project_templates_group_id)
REFERENCES namespaces (id)
ON DELETE SET NULL
NOT VALID;
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migration_helpers.rb:192:in `add_concurrent_foreign_key'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:237:in `add_concurrent_foreign_key'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:1899:in `add_missing_foreign_keys'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:149:in `up'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:56:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
ActiveRecord::StatementInvalid: PG::IndexCorrupted: ERROR: index "pg_trigger_tgconstraint_index" contains unexpected zero page at block 0
HINT: Please REINDEX it.
: ALTER TABLE application_settings
ADD CONSTRAINT fk_rails_b53e481273
FOREIGN KEY (custom_project_templates_group_id)
REFERENCES namespaces (id)
ON DELETE SET NULL
NOT VALID;
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migration_helpers.rb:192:in `add_concurrent_foreign_key'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:237:in `add_concurrent_foreign_key'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:1899:in `add_missing_foreign_keys'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:149:in `up'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:56:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
PG::IndexCorrupted: ERROR: index "pg_trigger_tgconstraint_index" contains unexpected zero page at block 0
HINT: Please REINDEX it.
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migration_helpers.rb:192:in `add_concurrent_foreign_key'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:237:in `add_concurrent_foreign_key'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:1899:in `add_missing_foreign_keys'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:149:in `up'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:56:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Tasks: TOP => db:migrate
(See full trace by running task with --trace)
-- transaction_open?()
-> 0.0000s
-- index_exists?(:users, :support_bot, {:algorithm=>:concurrently})
-> 0.0084s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:users, :bot_type, {:algorithm=>:concurrently})
-> 0.0082s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:users, :state, {:name=>"index_users_on_state_and_internal_attrs", :where=>"((ghost <> true) AND (support_bot <> true))", :algorithm=>:concurrently})
-> 0.0065s
-- transaction_open?()
-> 0.0000s
-- select_one("SELECT current_setting('server_version_num') AS v")
-> 0.0004s
-- index_exists?(:users, :state, {:name=>"index_users_on_state_and_internal", :algorithm=>:concurrently})
-> 0.0071s
-- execute("SET statement_timeout TO 0")
-> 0.0002s
-- remove_index(:users, {:name=>"index_users_on_state_and_internal", :algorithm=>:concurrently, :column=>:state})
-> 0.0101s
-- execute("RESET ALL")
-> 0.0002s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:users, :state, {:name=>"index_users_on_state_and_internal", :where=>"((ghost <> true) AND (bot_type IS NULL))", :algorithm=>:concurrently})
-> 0.0080s
-- execute("SET statement_timeout TO 0")
-> 0.0001s
-- add_index(:users, :state, {:name=>"index_users_on_state_and_internal", :where=>"((ghost <> true) AND (bot_type IS NULL))", :algorithm=>:concurrently})
-> 0.0079s
-- execute("RESET ALL")
-> 0.0001s
-- column_exists?(:web_hooks, :group_id)
-> 0.0021s
-- column_exists?(:geo_nodes, :internal_url)
-> 0.0014s
-- foreign_keys("application_settings")
-> 0.0050s
-- transaction_open?()
-> 0.0000s
-- foreign_keys("application_settings")
-> 0.0025s
-- execute("ALTER TABLE application_settings\nADD CONSTRAINT fk_rails_b53e481273\nFOREIGN KEY (custom_project_templates_group_id)\nREFERENCES namespaces (id)\nON DELETE SET NULL\nNOT VALID;\n")
STDERR:
---- End output of "bash" "/tmp/chef-script20190629-27-1084s7q" ----
Ran "bash" "/tmp/chef-script20190629-27-1084s7q" returned 1
Resource Declaration:
---------------------
# In /opt/gitlab/embedded/cookbooks/cache/cookbooks/gitlab/recipes/database_migrations.rb
53: bash "migrate gitlab-rails database" do
54: code <<-EOH
55: set -e
56: log_file="#{node['gitlab']['gitlab-rails']['log_directory']}/gitlab-rails-db-migrate-$(date +%Y-%m-%d-%H-%M-%S).log"
57: umask 077
58: /opt/gitlab/bin/gitlab-rake gitlab:db:configure 2>& 1 | tee ${log_file}
59: STATUS=${PIPESTATUS[0]}
60: echo $STATUS > #{db_migrate_status_file}
61: exit $STATUS
62: EOH
63: environment env_variables unless env_variables.empty?
64: notifies :run, "execute[clear the gitlab-rails cache]", :immediately
65: dependent_services.each do |svc|
66: notifies :restart, svc, :immediately
67: end
68: not_if "(test -f #{db_migrate_status_file}) && (cat #{db_migrate_status_file} | grep -Fx 0)"
69: only_if { node['gitlab']['gitlab-rails']['auto_migrate'] }
70: end
Compiled Resource:
------------------
# Declared in /opt/gitlab/embedded/cookbooks/cache/cookbooks/gitlab/recipes/database_migrations.rb:53:in `from_file'
bash("migrate gitlab-rails database") do
action [:run]
default_guard_interpreter :default
command nil
backup 5
returns 0
user nil
interpreter "bash"
declared_type :bash
cookbook_name "gitlab"
recipe_name "database_migrations"
code " set -e\n log_file=\"/var/log/gitlab/gitlab-rails/gitlab-rails-db-migrate-$(date +%Y-%m-%d-%H-%M-%S).log\"\n umask 077\n /opt/gitlab/bin/gitlab-rake gitlab:db:configure 2>& 1 | tee ${log_file}\n STATUS=${PIPESTATUS[0]}\n echo $STATUS > /var/opt/gitlab/gitlab-rails/upgrade-status/db-migrate-873248b1f0d3a7a5535771a3a1635803-1a9fd38a4ca\n exit $STATUS\n"
domain nil
not_if "(test -f /var/opt/gitlab/gitlab-rails/upgrade-status/db-migrate-873248b1f0d3a7a5535771a3a1635803-1a9fd38a4ca) && (cat /var/opt/gitlab/gitlab-rails/upgrade-status/db-migrate-873248b1f0d3a7a5535771a3a1635803-1a9fd38a4ca | grep -Fx 0)"
only_if { #code block }
end
System Info:
------------
chef_version=13.6.4
platform=ubuntu
platform_version=16.04
ruby=ruby 2.6.3p62 (2019-04-16 revision 67580) [x86_64-linux]
program_name=/opt/gitlab/embedded/bin/chef-client
executable=/opt/gitlab/embedded/bin/chef-client
Running handlers:
There was an error running gitlab-ctl reconfigure:
bash[migrate gitlab-rails database] (gitlab::database_migrations line 53) had an error: Mixlib::ShellOut::ShellCommandFailed: Expected process to exit with [0], but received '1'
---- Begin output of "bash" "/tmp/chef-script20190629-27-1084s7q" ----
STDOUT: == 20190402150158 BackportEnterpriseSchema: migrating =========================
-- column_exists?(:epics, :parent_id)
-> 0.0020s
-- column_exists?(:geo_event_log, :cache_invalidation_event_id)
-> 0.0014s
-- column_exists?(:vulnerability_feedback, :merge_request_id)
-> 0.0011s
-- column_exists?(:appearances, :updated_by)
-> 0.0013s
-- change_column_null(:application_settings, :password_authentication_enabled_for_git, false)
-> 0.0014s
-- change_table(:application_settings, {})
-> 0.4129s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:application_settings, :custom_project_templates_group_id, {:algorithm=>:concurrently})
-> 0.0036s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:application_settings, :file_template_project_id, {:algorithm=>:concurrently})
-> 0.0017s
-- column_exists?(:boards, :milestone_id)
-> 0.0012s
-- column_exists?(:boards, :weight)
-> 0.0010s
-- column_exists?(:boards, :name)
-> 0.0010s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:boards, :milestone_id, {:algorithm=>:concurrently})
-> 0.0018s
-- column_exists?(:clusters_applications_prometheus, :encrypted_alert_manager_token)
-> 0.0012s
-- column_exists?(:clusters_applications_prometheus, :encrypted_alert_manager_token_iv)
-> 0.0012s
-- column_exists?(:clusters_applications_prometheus, :last_update_started_at)
-> 0.0010s
-- column_exists?(:identities, :saml_provider_id)
-> 0.0010s
-- column_exists?(:identities, :secondary_extern_uid)
-> 0.0009s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:identities, :saml_provider_id, {:where=>"saml_provider_id IS NOT NULL", :algorithm=>:concurrently})
-> 0.0014s
-- column_exists?(:issues, :service_desk_reply_to)
-> 0.0025s
-- column_exists?(:issues, :weight)
-> 0.0017s
-- column_exists?(:lists, :milestone_id)
-> 0.0011s
-- column_exists?(:lists, :user_id)
-> 0.0010s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:lists, :milestone_id, {:algorithm=>:concurrently})
-> 0.0022s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:lists, :user_id, {:algorithm=>:concurrently})
-> 0.0022s
-- column_exists?(:members, :ldap)
-> 0.0013s
-- column_exists?(:members, :override)
-> 0.0012s
-- column_exists?(:merge_requests, :approvals_before_merge)
-> 0.0022s
-- column_exists?(:notes, :review_id)
-> 0.0018s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:notes, :review_id, {:algorithm=>:concurrently})
-> 0.0038s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:ci_builds, [:name], {:name=>"index_ci_builds_on_name_for_security_products_values", :where=>"\n (\n (name)::text = ANY (\n ARRAY[\n ('container_scanning'::character varying)::text,\n ('dast'::character varying)::text,\n ('dependency_scanning'::character varying)::text,\n ('license_management'::character varying)::text,\n ('sast'::character varying)::text\n ]\n )\n )", :algorithm=>:concurrently})
-> 0.0070s
-- index_exists?(:environments, :name, {:name=>"index_environments_on_name_varchar_pattern_ops"})
-> 0.0022s
-- change_table(:namespaces, {})
-> 0.0326s
-- column_exists?(:namespaces, :ldap_sync_status)
-> 0.0025s
-- column_exists?(:namespaces, :membership_lock)
-> 0.0019s
-- indexes(:namespaces)
-> 0.0064s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:namespaces, :file_template_project_id, {:algorithm=>:concurrently})
-> 0.0076s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:namespaces, :ldap_sync_last_successful_update_at, {:algorithm=>:concurrently})
-> 0.0054s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:namespaces, :ldap_sync_last_update_at, {:algorithm=>:concurrently})
-> 0.0054s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:namespaces, :plan_id, {:algorithm=>:concurrently})
-> 0.0053s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:namespaces, :trial_ends_on, {:where=>"(trial_ends_on IS NOT NULL)", :algorithm=>:concurrently})
-> 0.0060s
-- indexes(:namespaces)
-> 0.0058s
-- column_exists?(:notification_settings, :new_epic)
-> 0.0019s
-- change_table(:project_mirror_data, {})
-> 0.0052s
-- column_exists?(:project_mirror_data, :retry_count)
-> 0.0010s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:project_mirror_data, :last_successful_update_at, {:algorithm=>:concurrently})
-> 0.0021s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:project_mirror_data, [:next_execution_timestamp, :retry_count], {:name=>"index_mirror_data_on_next_execution_and_retry_count", :algorithm=>:concurrently})
-> 0.0019s
-- column_exists?(:project_statistics, :shared_runners_seconds)
-> 0.0011s
-- column_exists?(:project_statistics, :shared_runners_seconds_last_reset)
-> 0.0010s
-- change_table(:projects, {})
-> 0.0802s
-- change_column_null(:projects, :merge_requests_rebase_enabled, true)
-> 0.0007s
-- column_exists?(:projects, :mirror)
-> 0.0036s
-- column_exists?(:projects, :mirror_trigger_builds)
-> 0.0031s
-- column_exists?(:projects, :reset_approvals_on_push)
-> 0.0039s
-- column_exists?(:projects, :service_desk_enabled)
-> 0.0031s
-- column_exists?(:projects, :approvals_before_merge)
-> 0.0031s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:projects, [:archived, :pending_delete, :merge_requests_require_code_owner_approval], {:name=>"projects_requiring_code_owner_approval", :where=>"((pending_delete = false) AND (archived = false) AND (merge_requests_require_code_owner_approval = true))", :algorithm=>:concurrently})
-> 0.0085s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:projects, [:id, :repository_storage, :last_repository_updated_at], {:name=>"idx_projects_on_repository_storage_last_repository_updated_at", :algorithm=>:concurrently})
-> 0.0079s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:projects, :id, {:name=>"index_projects_on_mirror_and_mirror_trigger_builds_both_true", :where=>"((mirror IS TRUE) AND (mirror_trigger_builds IS TRUE))", :algorithm=>:concurrently})
-> 0.0077s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:projects, :mirror_last_successful_update_at, {:algorithm=>:concurrently})
-> 0.0075s
-- change_column_null(:protected_branch_merge_access_levels, :access_level, true)
-> 0.0004s
-- column_exists?(:protected_branch_merge_access_levels, :group_id)
-> 0.0011s
-- column_exists?(:protected_branch_merge_access_levels, :user_id)
-> 0.0009s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:protected_branch_merge_access_levels, :group_id, {:algorithm=>:concurrently})
-> 0.0014s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:protected_branch_merge_access_levels, :user_id, {:algorithm=>:concurrently})
-> 0.0014s
-- change_column_null(:protected_branch_push_access_levels, :access_level, true)
-> 0.0003s
-- column_exists?(:protected_branch_push_access_levels, :group_id)
-> 0.0009s
-- column_exists?(:protected_branch_push_access_levels, :user_id)
-> 0.0009s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:protected_branch_push_access_levels, :group_id, {:algorithm=>:concurrently})
-> 0.0015s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:protected_branch_push_access_levels, :user_id, {:algorithm=>:concurrently})
-> 0.0013s
-- column_exists?(:resource_label_events, :epic_id)
-> 0.0012s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:resource_label_events, :epic_id, {:algorithm=>:concurrently})
-> 0.0019s
-- column_exists?(:user_preferences, :epic_notes_filter)
-> 0.0012s
-- column_exists?(:user_preferences, :epics_sort)
-> 0.0011s
-- column_exists?(:user_preferences, :roadmap_epics_state)
-> 0.0011s
-- column_exists?(:user_preferences, :roadmaps_sort)
-> 0.0011s
-- column_exists?(:users, :auditor)
-> 0.0034s
-- change_table(:users, {})
-> 0.0376s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:users, :group_view, {:algorithm=>:concurrently})
-> 0.0071s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:users, :managing_group_id, {:algorithm=>:concurrently})
-> 0.0067srake aborted!
StandardError: An error has occurred, all later migrations canceled:
PG::IndexCorrupted: ERROR: index "pg_trigger_tgconstraint_index" contains unexpected zero page at block 0
HINT: Please REINDEX it.
: ALTER TABLE application_settings
ADD CONSTRAINT fk_rails_b53e481273
FOREIGN KEY (custom_project_templates_group_id)
REFERENCES namespaces (id)
ON DELETE SET NULL
NOT VALID;
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migration_helpers.rb:192:in `add_concurrent_foreign_key'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:237:in `add_concurrent_foreign_key'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:1899:in `add_missing_foreign_keys'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:149:in `up'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:56:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
ActiveRecord::StatementInvalid: PG::IndexCorrupted: ERROR: index "pg_trigger_tgconstraint_index" contains unexpected zero page at block 0
HINT: Please REINDEX it.
: ALTER TABLE application_settings
ADD CONSTRAINT fk_rails_b53e481273
FOREIGN KEY (custom_project_templates_group_id)
REFERENCES namespaces (id)
ON DELETE SET NULL
NOT VALID;
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migration_helpers.rb:192:in `add_concurrent_foreign_key'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:237:in `add_concurrent_foreign_key'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:1899:in `add_missing_foreign_keys'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:149:in `up'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:56:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Caused by:
PG::IndexCorrupted: ERROR: index "pg_trigger_tgconstraint_index" contains unexpected zero page at block 0
HINT: Please REINDEX it.
/opt/gitlab/embedded/service/gitlab-rails/lib/gitlab/database/migration_helpers.rb:192:in `add_concurrent_foreign_key'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:237:in `add_concurrent_foreign_key'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:1899:in `add_missing_foreign_keys'
/opt/gitlab/embedded/service/gitlab-rails/db/migrate/20190402150158_backport_enterprise_schema.rb:149:in `up'
/opt/gitlab/embedded/service/gitlab-rails/lib/tasks/gitlab/db.rake:56:in `block (3 levels) in <top (required)>'
/opt/gitlab/embedded/bin/bundle:23:in `load'
/opt/gitlab/embedded/bin/bundle:23:in `<main>'
Tasks: TOP => db:migrate
(See full trace by running task with --trace)
-- transaction_open?()
-> 0.0000s
-- index_exists?(:users, :support_bot, {:algorithm=>:concurrently})
-> 0.0084s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:users, :bot_type, {:algorithm=>:concurrently})
-> 0.0082s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:users, :state, {:name=>"index_users_on_state_and_internal_attrs", :where=>"((ghost <> true) AND (support_bot <> true))", :algorithm=>:concurrently})
-> 0.0065s
-- transaction_open?()
-> 0.0000s
-- select_one("SELECT current_setting('server_version_num') AS v")
-> 0.0004s
-- index_exists?(:users, :state, {:name=>"index_users_on_state_and_internal", :algorithm=>:concurrently})
-> 0.0071s
-- execute("SET statement_timeout TO 0")
-> 0.0002s
-- remove_index(:users, {:name=>"index_users_on_state_and_internal", :algorithm=>:concurrently, :column=>:state})
-> 0.0101s
-- execute("RESET ALL")
-> 0.0002s
-- transaction_open?()
-> 0.0000s
-- index_exists?(:users, :state, {:name=>"index_users_on_state_and_internal", :where=>"((ghost <> true) AND (bot_type IS NULL))", :algorithm=>:concurrently})
-> 0.0080s
-- execute("SET statement_timeout TO 0")
-> 0.0001s
-- add_index(:users, :state, {:name=>"index_users_on_state_and_internal", :where=>"((ghost <> true) AND (bot_type IS NULL))", :algorithm=>:concurrently})
-> 0.0079s
-- execute("RESET ALL")
-> 0.0001s
-- column_exists?(:web_hooks, :group_id)
-> 0.0021s
-- column_exists?(:geo_nodes, :internal_url)
-> 0.0014s
-- foreign_keys("application_settings")
-> 0.0050s
-- transaction_open?()
-> 0.0000s
-- foreign_keys("application_settings")
-> 0.0025s
-- execute("ALTER TABLE application_settings\nADD CONSTRAINT fk_rails_b53e481273\nFOREIGN KEY (custom_project_templates_group_id)\nREFERENCES namespaces (id)\nON DELETE SET NULL\nNOT VALID;\n")
STDERR:
---- End output of "bash" "/tmp/chef-script20190629-27-1084s7q" ----
Ran "bash" "/tmp/chef-script20190629-27-1084s7q" returned 1
Running handlers complete
Chef Client failed. 9 resources updated in 30 seconds
Please notice that this is only the error part of the output.
After failing Chef, I noticed that docker stopped that container, started and tried again automatically (while kicking me out of the container).
Because of this behavior, I assume that docker is working correctly.
Also, I am quite unsure whether this is a bug or not. On the other hand, I pretty much used standard configuration.
Could you please give me advice on how to resolve that migration error or which additional information would be needed to resolve it.
In the meantime, I could manage to load a working backup of the whole machine, so I can recreate every step if needed.
Thank you.
Firebird