[ Feb 22 01:10:44 Enabled. ]
[ Feb 22 01:10:44 Rereading configuration. ]
[ Feb 22 01:10:46 Rereading configuration. ]
[ Feb 22 01:10:49 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:10:49.332454946Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:10:49.333531387Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:10:49.333541747Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:10:49.48757188Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:42.946005736Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:35930"}
{"msg":"region 3ea0fab0-2d62-4972-bc8b-83c2de32c822 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:42.948235391Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:42.949424765Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"uri":"/crucible/0/regions","method":"POST","req_id":"17ce0720-dfd3-49a9-a8e3-32b381560ab2","remote_addr":"[fd00:1122:3344:101::b]:35930","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":2053,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:42.950094675Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/3ea0fab0-2d62-4972-bc8b-83c2de32c822","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:42.959852958Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/3ea0fab0-2d62-4972-bc8b-83c2de32c822","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:42.959890109Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3ea0fab0-2d62-4972-bc8b-83c2de32c822\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/3ea0fab0-2d62-4972-bc8b-83c2de32c822\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.030133709Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"region":"3ea0fab0-2d62-4972-bc8b-83c2de32c822","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.123990165Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"uri":"/crucible/0/regions","method":"POST","req_id":"21bef02d-37f9-4f9b-83ba-381277d89388","remote_addr":"[fd00:1122:3344:101::b]:35930","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.149103264Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"region":"3ea0fab0-2d62-4972-bc8b-83c2de32c822","component":"worker"}
{"msg":"region 3ea0fab0-2d62-4972-bc8b-83c2de32c822 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.149133195Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.149266688Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3ea0fab0-2d62-4972-bc8b-83c2de32c822","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.149757023Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3ea0fab0-2d62-4972-bc8b-83c2de32c822","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.154017016Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.161389591Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3ea0fab0-2d62-4972-bc8b-83c2de32c822","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.164214133Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3ea0fab0-2d62-4972-bc8b-83c2de32c822","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.164770209Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.164908533Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.164954444Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.164994545Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.168847777Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.173854242Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.391022859Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"uri":"/crucible/0/regions","method":"POST","req_id":"a716c7eb-e704-4a5e-8a81-d36238628770","remote_addr":"[fd00:1122:3344:101::b]:35930","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.314151343Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:38189"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.314328138Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"uri":"/crucible/0/regions/3ea0fab0-2d62-4972-bc8b-83c2de32c822","method":"GET","req_id":"f459545a-438b-450a-844c-47e719f9281b","remote_addr":"[fd00:1122:3344:101::b]:38189","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":118,"response_code":"200"}
{"msg":"path is \"/data/regions/3ea0fab0-2d62-4972-bc8b-83c2de32c822\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.314575725Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"datafile"}
{"msg":"dataset is oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/3ea0fab0-2d62-4972-bc8b-83c2de32c822","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.32438836Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"datafile"}
{"msg":"region 3ea0fab0-2d62-4972-bc8b-83c2de32c822 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.333437223Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.333581267Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.333646509Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"uri":"/crucible/0/regions/3ea0fab0-2d62-4972-bc8b-83c2de32c822","method":"DELETE","req_id":"0e3397a6-dead-4744-9373-d89ea63663ed","remote_addr":"[fd00:1122:3344:101::b]:38189","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":19111,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.333792634Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"uri":"/crucible/0/regions/3ea0fab0-2d62-4972-bc8b-83c2de32c822","method":"GET","req_id":"d64bfada-a9da-49e3-989a-f3512ead2a10","remote_addr":"[fd00:1122:3344:101::b]:38189","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":21,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-3ea0fab0-2d62-4972-bc8b-83c2de32c822 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.334652028Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.338043737Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/3ea0fab0-2d62-4972-bc8b-83c2de32c822\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.34850279Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"region":"3ea0fab0-2d62-4972-bc8b-83c2de32c822","component":"worker"}
{"msg":"region 3ea0fab0-2d62-4972-bc8b-83c2de32c822 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.400932343Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.415107745Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"uri":"/crucible/0/regions/3ea0fab0-2d62-4972-bc8b-83c2de32c822","method":"GET","req_id":"da363a1e-2797-49cd-a9d3-36011cbdde56","remote_addr":"[fd00:1122:3344:101::b]:38189","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.202131484Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:54598"}
{"msg":"region 90bbc30a-2152-462a-abd9-8748cfbcf1f5 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.232402773Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.23264368Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"uri":"/crucible/0/regions","method":"POST","req_id":"ee0df10a-0a7b-4259-9392-05cf2abac6de","remote_addr":"[fd00:1122:3344:101::a]:54598","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":292,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.232702192Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/90bbc30a-2152-462a-abd9-8748cfbcf1f5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.242524887Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crucible/regions/90bbc30a-2152-462a-abd9-8748cfbcf1f5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.242586329Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"90bbc30a-2152-462a-abd9-8748cfbcf1f5\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/90bbc30a-2152-462a-abd9-8748cfbcf1f5\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.312965133Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"region":"90bbc30a-2152-462a-abd9-8748cfbcf1f5","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.442129913Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"region":"90bbc30a-2152-462a-abd9-8748cfbcf1f5","component":"worker"}
{"msg":"region 90bbc30a-2152-462a-abd9-8748cfbcf1f5 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.442184415Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.442269767Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-90bbc30a-2152-462a-abd9-8748cfbcf1f5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.443638037Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-90bbc30a-2152-462a-abd9-8748cfbcf1f5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.451701791Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.470366193Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-90bbc30a-2152-462a-abd9-8748cfbcf1f5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.480880589Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/90bbc30a-2152-462a-abd9-8748cfbcf1f5","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.481125176Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.481188557Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.481232609Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.48127478Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.48954541Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.497708197Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:12:25.499351275Z","hostname":"oxz_crucible_ce42aa8c-6fb7-4d71-931b-8d3f1804ce2a","pid":23522,"uri":"/crucible/0/regions","method":"POST","req_id":"cf33a849-7400-4937-ba58-47b7c7869b8f","remote_addr":"[fd00:1122:3344:101::a]:54598","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":65,"response_code":"200"}