MongoDB | 2659590

Info

Hi Team, We initiated Y2038 testing by advancing the system clock to 2038-01-19T01:00:00. Initially, the replicas were functioning healthily, servicing traffic without issues. However, as the system approached the Y2038 time threshold, MongoDB experienced a sudden crash, accompanied by the following error messages in the primary node Test Procedure is simple 1) Have a PSA 2) set the clock to 2038-01-19T01:00:00 and observe the replicaSet health 3) once the system clock moves beyond  2038-01-19T03:14:07 observe the crash We are using v5.0.20 {"t":\{"$date":"2038-01-19T03:14:10.057+00:00"} ,"s":"I",  "c":"NETWORK",  "id":51800, "ctx":"conn13180","msg":"client metadata","attr":{"remote":"172.20.35.88:54892","client":"conn13180","doc":{"application": {"name":"MongoDB Shell"} ,"driver":{"name":"MongoDB Internal Client","version":"5.0.20"},"os":{"type":"Linux","name":"AlmaLinux release 8.8 (Sapphire Caracal)","architecture":"x86_64","version":"Kernel 4.18.0-477.27.1.el8_8.x86_64"}}}} {"t":\{"$date":"2038-01-19T03:14:10.090+00:00"} ,"s":"I",  "c":"-", "id":20883, "ctx":"conn13178","msg":"Interrupted operation as its client disconnected","attr":{"opId":230177}} {"t":\{"$date":"2038-01-19T03:14:10.109+00:00"} ,"s":"I",  "c":"NETWORK",  "id":51800, "ctx":"conn13181","msg":"client metadata","attr":{"remote":"172.20.35.88:54906","client":"conn13181","doc":{"application": {"name":"MongoDB Shell"} ,"driver":{"name":"MongoDB Internal Client","version":"5.0.20"},"os":{"type":"Linux","name":"AlmaLinux release 8.8 (Sapphire Caracal)","architecture":"x86_64","version":"Kernel 4.18.0-477.27.1.el8_8.x86_64"}}}} {"t":\{"$date":"2038-01-19T03:14:10.233+00:00"} ,"s":"F",  "c":"CONTROL",  "id":6384300, "ctx":"NoopWriter","msg":"Writing fatal message","attr":{"message":"terminate() called. An exception is active; attempting to gather more information\n"}} {"t":\{"$date":"2038-01-19T03:14:10.233+00:00"} ,"s":"F",  "c":"CONTROL",  "id":6384300, "ctx":"NoopWriter","msg":"Writing fatal message","attr":{"message":"DBException::toString(): Location40482: $clusterTime cannot be advanced beyond the maximum logical time value\nActual exception type: mongo::error_details::throwExceptionForStatus(mongo::Status const&)::NonspecificAssertionException\n\n"}} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31380, "ctx":"NoopWriter","msg":"BACKTRACE","attr":{"bt":{"backtrace":[ {"a":"55B48376E365","b":"55B47F7F2000","o":"3F7C365","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.361","s+":"215"} ,{"a":"55B483770DE9","b":"55B47F7F2000","o":"3F7EDE9","s":"ZN5mongo15printStackTraceEv","s+":"29"},{"a":"55B483769907","b":"55B47F7F2000","o":"3F77907","s":"_ZN5mongo12_GLOBAL_N_111myTerminateEv","s+":"D7"},{"a":"55B483903BB6","b":"55B47F7F2000","o":"4111BB6","s":"ZN10cxxabiv111terminateEPFvvE","s+":"6"},{"a":"55B483903BF1","b":"55B47F7F2000","o":"4111BF1","s":"_ZSt9terminatev","s+":"11"},{"a":"55B483903D24","b":"55B47F7F2000","o":"4111D24","s":"cxa_throw","s+":"44"},{"a":"55B480BF6B2F","b":"55B47F7F2000","o":"1404B2F","s":"_ZN5mongo13error_details23throwExceptionForStatusERKNS_6StatusE","s+":"8DCF"},{"a":"55B480C068B3","b":"55B47F7F2000","o":"14148B3","s":"_ZN5mongo21uassertedWithLocationERKNS_6StatusEPKcj","s+":"337"},{"a":"55B480A3A571","b":"55B47F7F2000","o":"1248571","s":"_ZN5mongo18VectorClockMutable28_advanceComponentTimeByTicksENS_11VectorClock9ComponentEm.cold.404","s+":"93"},{"a":"55B4824352FE","b":"55B47F7F2000","o":"2C432FE","s":"_ZN5mongo14LocalOplogInfo14getNextOpTimesEPNS_16OperationContextEm","s+":"AE"},{"a":"55B481C56D88","b":"55B47F7F2000","o":"2464D88","s":"_ZN5mongo4repl5logOpEPNS_16OperationContextEPNS0_17MutableOplogEntryE","s+":"158"},{"a":"55B4816777DB","b":"55B47F7F2000","o":"1E857DB","s":"_ZN5mongo12_GLOBAL_N_112logOperationEPNS_16OperationContextEPNS_4repl17MutableOplogEntryE","s+":"4B"},{"a":"55B48167B861","b":"55B47F7F2000","o":"1E89861","s":"ZN5mongo14OpObserverImpl19onInternalOpMessageEPNS_16OperationContextERKNS_15NamespaceStringEN5boost8optionalINS_4UUIDEEERKNS_7BSONObjENS7_ISA_EENS7_INS_4repl6OpTimeEEESG_SG_SG","s+":"231"},{"a":"55B480D90CCA","b":"55B47F7F2000","o":"159ECCA","s":"ZN5mongo18OpObserverRegistry19onInternalOpMessageEPNS_16OperationContextERKNS_15NamespaceStringEN5boost8optionalINS_4UUIDEEERKNS_7BSONObjENS7_ISA_EENS7_INS_4repl6OpTimeEEESG_SG_SG","s+":"21A"},{"a":"55B4810B18C7","b":"55B47F7F2000","o":"18BF8C7","s":"_ZZN5mongo4repl10NoopWriter10_writeNoopEPNS_16OperationContextEENKUlvE3_clEv.isra.299","s+":"D7"},{"a":"55B4810B2069","b":"55B47F7F2000","o":"18C0069","s":"_ZN5mongo4repl10NoopWriter10_writeNoopEPNS_16OperationContextE","s+":"719"},{"a":"55B4810B367B","b":"55B47F7F2000","o":"18C167B","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_4repl10NoopWriter18PeriodicNoopRunnerC4ENS3_8DurationISt5ratioILl1ELl1EEEESt8functionIFvPNS3_16OperationContextEEEEUlvE_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"3BB"},{"a":"55B48391FE4F","b":"55B47F7F2000","o":"412DE4F","s":"execute_native_thread_routine","s+":"F"},{"a":"7FCBBDD811CA","b":"7FCBBDD79000","o":"81CA","s":"start_thread","s+":"EA"},{"a":"7FCBBD9EDE73","b":"7FCBBD9B4000","o":"39E73","s":"clone","s+":"43"}],"processInfo":{"mongodbVersion":"5.0.20","gitVersion":"2cd626d8148120319d7dca5824e760fe220cb0de","compiledModules":[],"uname": {"sysname":"Linux","release":"4.18.0-477.27.1.el8_8.x86_64","version":"#1 SMP Thu Sep 21 06:49:25 EDT 2023","machine":"x86_64"} ,"somap":[\{"b":"55B47F7F2000","elfType":3,"buildId":"A8EA7166EFC23E0D3802F8AFEFEF2186CF5E5BBD"},\{"b":"7FCBBDD79000","path":"/lib64/libpthread.so.0","elfType":3,"buildId":"76F163FDBAA9E91050B456A7E5EA8AC78563BD29"},\{"b":"7FCBBD9B4000","path":"/lib64/libc.so.6","elfType":3,"buildId":"44ED73CF68E8FA608DA3B301146C81A0A77A5619"}]}}}} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B48376E365","b":"55B47F7F2000","o":"3F7C365","s":"_ZN5mongo18stack_trace_detail12_GLOBAL__N_119printStackTraceImplERKNS1_7OptionsEPNS_14StackTraceSinkE.constprop.361","s+":"215"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B483770DE9","b":"55B47F7F2000","o":"3F7EDE9","s":"_ZN5mongo15printStackTraceEv","s+":"29"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B483769907","b":"55B47F7F2000","o":"3F77907","s":"_ZN5mongo12_GLOBAL__N_111myTerminateEv","s+":"D7"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B483903BB6","b":"55B47F7F2000","o":"4111BB6","s":"_ZN10__cxxabiv111__terminateEPFvvE","s+":"6"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B483903BF1","b":"55B47F7F2000","o":"4111BF1","s":"_ZSt9terminatev","s+":"11"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B483903D24","b":"55B47F7F2000","o":"4111D24","s":"__cxa_throw","s+":"44"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B480BF6B2F","b":"55B47F7F2000","o":"1404B2F","s":"_ZN5mongo13error_details23throwExceptionForStatusERKNS_6StatusE","s+":"8DCF"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B480C068B3","b":"55B47F7F2000","o":"14148B3","s":"_ZN5mongo21uassertedWithLocationERKNS_6StatusEPKcj","s+":"337"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B480A3A571","b":"55B47F7F2000","o":"1248571","s":"_ZN5mongo18VectorClockMutable28_advanceComponentTimeByTicksENS_11VectorClock9ComponentEm.cold.404","s+":"93"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B4824352FE","b":"55B47F7F2000","o":"2C432FE","s":"_ZN5mongo14LocalOplogInfo14getNextOpTimesEPNS_16OperationContextEm","s+":"AE"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B481C56D88","b":"55B47F7F2000","o":"2464D88","s":"_ZN5mongo4repl5logOpEPNS_16OperationContextEPNS0_17MutableOplogEntryE","s+":"158"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B4816777DB","b":"55B47F7F2000","o":"1E857DB","s":"_ZN5mongo12_GLOBAL__N_112logOperationEPNS_16OperationContextEPNS_4repl17MutableOplogEntryE","s+":"4B"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B48167B861","b":"55B47F7F2000","o":"1E89861","s":"_ZN5mongo14OpObserverImpl19onInternalOpMessageEPNS_16OperationContextERKNS_15NamespaceStringEN5boost8optionalINS_4UUIDEEERKNS_7BSONObjENS7_ISA_EENS7_INS_4repl6OpTimeEEESG_SG_SG_","s+":"231"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B480D90CCA","b":"55B47F7F2000","o":"159ECCA","s":"_ZN5mongo18OpObserverRegistry19onInternalOpMessageEPNS_16OperationContextERKNS_15NamespaceStringEN5boost8optionalINS_4UUIDEEERKNS_7BSONObjENS7_ISA_EENS7_INS_4repl6OpTimeEEESG_SG_SG_","s+":"21A"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B4810B18C7","b":"55B47F7F2000","o":"18BF8C7","s":"_ZZN5mongo4repl10NoopWriter10_writeNoopEPNS_16OperationContextEENKUlvE3_clEv.isra.299","s+":"D7"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B4810B2069","b":"55B47F7F2000","o":"18C0069","s":"_ZN5mongo4repl10NoopWriter10_writeNoopEPNS_16OperationContextE","s+":"719"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B4810B367B","b":"55B47F7F2000","o":"18C167B","s":"_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN5mongo4stdx6threadC4IZNS3_4repl10NoopWriter18PeriodicNoopRunnerC4ENS3_8DurationISt5ratioILl1ELl1EEEESt8functionIFvPNS3_16OperationContextEEEEUlvE_JELi0EEET_DpOT0_EUlvE_EEEEE6_M_runEv","s+":"3BB"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"55B48391FE4F","b":"55B47F7F2000","o":"412DE4F","s":"execute_native_thread_routine","s+":"F"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"7FCBBDD811CA","b":"7FCBBDD79000","o":"81CA","s":"start_thread","s+":"EA"} }} {"t":\{"$date":"2038-01-19T03:14:10.360+00:00"} ,"s":"I",  "c":"CONTROL",  "id":31445, "ctx":"NoopWriter","msg":"Frame","attr":{"frame": {"a":"7FCBBD9EDE73","b":"7FCBBD9B4000","o":"39E73","s":"clone","s+":"43"} }} {"t":\{"$date":"2038-01-19T03:14:18.915+00:00"} ,"s":"I",  "c":"CONTROL",  "id":20698, "ctx":"-","msg":"**** SERVER RESTARTED ****"} db.serverBuildInfo() { “version” : “5.0.20”, “gitVersion” : “2cd626d8148120319d7dca5824e760fe220cb0de”, “modules” : , “allocator” : “tcmalloc”, “javascriptEngine” : “mozjs”, “sysInfo” : “deprecated”, “versionArray” : [ 5, 0, 20, 0 ], “openssl” : { “running” : “OpenSSL 1.1.1k FIPS 25 Mar 2021”, “compiled” : “OpenSSL 1.1.1 FIPS 11 Sep 2018” } , “buildEnvironment” : { “distmod” : “rhel80”, “distarch” : “x86_64”, “cc” : “/opt/mongodbtoolchain/v3/bin/gcc: gcc (GCC) 8.5.0”, “ccflags” : “-Werror -include mongo/platform/basic.h -ffp-contract=off -fasynchronous-unwind-tables -ggdb -Wall -Wsign-compare -Wno-unknown-pragmas -Winvalid-pch -fno-omit-frame-pointer -fno-strict-aliasing -O2 -march=sandybridge -mtune=generic -mprefer-vector-width=128 -Wno-unused-local-typedefs -Wno-unused-function -Wno-deprecated-declarations -Wno-unused-const-variable -Wno-unused-but-set-variable -Wno-missing-braces -fstack-protector-strong -Wa,–nocompress-debug-sections -fno-builtin-memcmp”, “cxx” : “/opt/mongodbtoolchain/v3/bin/g+: g+ (GCC) 8.5.0”, “cxxflags” : “-Woverloaded-virtual -Wno-maybe-uninitialized -fsized-deallocation -std=c++17”, “linkflags” : “-Wl,–fatal-warnings -pthread -Wl,-z,now -fuse-ld=gold -fstack-protector-strong -Wl,–no-threads -Wl,–build-id -Wl,–hash-style=gnu -Wl,-z,noexecstack -Wl,–warn-execstack -Wl,-z,relro -Wl,–compress-debug-sections=none -Wl,-z,origin -Wl,–enable-new-dtags”, “target_arch” : “x86_64”, “target_os” : “linux”, “cppdefines” : “SAFEINT_USE_INTRINSICS 0 PCRE_STATIC NDEBUG _XOPEN_SOURCE 700 _GNU_SOURCE _FORTIFY_SOURCE 2 BOOST_THREAD_VERSION 5 BOOST_THREAD_USES_DATETIME BOOST_SYSTEM_NO_DEPRECATED BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS BOOST_ENABLE_ASSERT_DEBUG_HANDLER BOOST_LOG_NO_SHORTHAND_NAMES BOOST_LOG_USE_NATIVE_SYSLOG BOOST_LOG_WITHOUT_THREAD_ATTR ABSL_FORCE_ALIGNED_ACCESS” }, “bits” : 64, “debug” : false, “maxBsonObjectSize” : 16777216, “storageEngines” : [ “devnull”, “ephemeralForTest”, “wiredTiger” ], “ok” : 1 }

Top User Comments

JIRAUSER1258850 commented on Thu, 2 May 2024 06:30:35 +0000: Thank you, Chris Kelly. It appears that SERVER-36870 was opened in 2018, which was 6 years ago. While we understand that 2038 is still 14 years away, do you have any ETA for addressing this issue? We would like to assess the risks associated with other software as well. However, with MongoDB crashing itself, we are unable to proceed further on this matter. JIRAUSER1265262 commented on Wed, 1 May 2024 20:00:27 +0000: Hi veramasu@hcl.com, Thanks for your report! We currently have this backlogged in SERVER-36870.

Steps to Reproduce