[ Jan 10 21:56:19 Enabled. ]
[ Jan 10 21:56:19 Rereading configuration. ]
[ Jan 10 21:56:22 Rereading configuration. ]
[ Jan 10 21:56:28 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_eecab578-d379-46e4-90f5-9dc11839dad5/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_eecab578-d379-46e4-90f5-9dc11839dad5/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:56:28.50291257Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:56:28.506575285Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:56:28.506585896Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:56:28.523694748Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:56:28.790004527Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:25.815995697Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:56156"}
{"msg":"region ffbf93c5-4c8f-4d02-9b3d-0bebf643db45 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:25.840172702Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:25.84532842Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:25.850646213Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"uri":"/crucible/0/regions","method":"POST","req_id":"4bd99001-f1bb-4828-95c6-dbed789e5a14","remote_addr":"[fd00:1122:3344:101::b]:56156","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":14725,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_eecab578-d379-46e4-90f5-9dc11839dad5/crucible/regions/ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:25.85748593Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_eecab578-d379-46e4-90f5-9dc11839dad5/crucible/regions/ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:25.857504901Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ffbf93c5-4c8f-4d02-9b3d-0bebf643db45\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ffbf93c5-4c8f-4d02-9b3d-0bebf643db45\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:25.92492397Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"region":"ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ffbf93c5-4c8f-4d02-9b3d-0bebf643db45\" \"--data\" \"/data/regions/ffbf93c5-4c8f-4d02-9b3d-0bebf643db45\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:25.927057801Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"region":"ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.069109037Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"region":"ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","component":"worker"}
{"msg":"region ffbf93c5-4c8f-4d02-9b3d-0bebf643db45 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.069182899Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.069270111Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.069764606Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.072486634Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"uri":"/crucible/0/regions","method":"POST","req_id":"7bf3f236-e010-4057-ab2f-4fd3d16a2bce","remote_addr":"[fd00:1122:3344:101::b]:56156","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.084078247Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.094691943Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.098983506Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.09983236Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.099905032Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.099958664Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.100013216Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.104748292Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.110695803Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.541248656Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"uri":"/crucible/0/regions/ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","method":"GET","req_id":"c401c271-3c62-4635-9f2f-515b4e5e07d9","remote_addr":"[fd00:1122:3344:101::b]:56156","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":71,"response_code":"200"}
{"msg":"path is \"/data/regions/ffbf93c5-4c8f-4d02-9b3d-0bebf643db45\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.545046915Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"datafile"}
{"msg":"dataset is oxp_eecab578-d379-46e4-90f5-9dc11839dad5/crucible/regions/ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.555437334Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"datafile"}
{"msg":"region ffbf93c5-4c8f-4d02-9b3d-0bebf643db45 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.565875594Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.566042489Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.566102731Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"uri":"/crucible/0/regions/ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","method":"DELETE","req_id":"85baf9dc-a403-42aa-b2f3-c7b0bc963073","remote_addr":"[fd00:1122:3344:101::b]:56156","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":21074,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.568552571Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"uri":"/crucible/0/regions/ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","method":"GET","req_id":"4ed7a3b5-0094-4d7c-a0e5-44f67b2f0437","remote_addr":"[fd00:1122:3344:101::b]:56156","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":39,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-ffbf93c5-4c8f-4d02-9b3d-0bebf643db45 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.569529699Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.574974456Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_eecab578-d379-46e4-90f5-9dc11839dad5/crucible/regions/ffbf93c5-4c8f-4d02-9b3d-0bebf643db45\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.586327602Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"region":"ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","component":"worker"}
{"msg":"region ffbf93c5-4c8f-4d02-9b3d-0bebf643db45 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.657222882Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-10T21:57:26.659332452Z","hostname":"oxz_crucible_f6578145-7a5a-4961-a28a-6e01465004d6","pid":22918,"uri":"/crucible/0/regions/ffbf93c5-4c8f-4d02-9b3d-0bebf643db45","method":"GET","req_id":"1486555f-6351-4449-b622-5fa932c7b778","remote_addr":"[fd00:1122:3344:101::b]:56156","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":75,"response_code":"200"}