[ Mar 12 22:35:18 Enabled. ]
[ Mar 12 22:35:18 Rereading configuration. ]
[ Mar 12 22:35:20 Rereading configuration. ]
[ Mar 12 22:35:24 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_034b6895-4501-4234-bb8e-9c97c0521fb0/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_034b6895-4501-4234-bb8e-9c97c0521fb0/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:35:24.824740552Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736}
{"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:35:24.827666698Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:35:24.827675228Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:35:24.843522459Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:35:25.276562643Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.593906535Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63558"}
{"msg":"region eb70d53b-bced-473e-b702-76e766617a32 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.630201556Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.632309481Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"uri":"/crucible/0/regions","method":"POST","req_id":"f1bda754-1212-43fc-ac84-9f59fc115b2a","remote_addr":"[fd00:1122:3344:101::a]:63558","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":17243,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.633096021Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_034b6895-4501-4234-bb8e-9c97c0521fb0/crucible/regions/eb70d53b-bced-473e-b702-76e766617a32","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.642991378Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_034b6895-4501-4234-bb8e-9c97c0521fb0/crucible/regions/eb70d53b-bced-473e-b702-76e766617a32","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.643027999Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"eb70d53b-bced-473e-b702-76e766617a32\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/eb70d53b-bced-473e-b702-76e766617a32\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.697999713Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"region":"eb70d53b-bced-473e-b702-76e766617a32","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"eb70d53b-bced-473e-b702-76e766617a32\" \"--data\" \"/data/regions/eb70d53b-bced-473e-b702-76e766617a32\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.698061125Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"region":"eb70d53b-bced-473e-b702-76e766617a32","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.827347006Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"region":"eb70d53b-bced-473e-b702-76e766617a32","component":"worker"}
{"msg":"region eb70d53b-bced-473e-b702-76e766617a32 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.827408657Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.82749522Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-eb70d53b-bced-473e-b702-76e766617a32","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.827962462Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-eb70d53b-bced-473e-b702-76e766617a32","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.838643838Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.848907944Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-eb70d53b-bced-473e-b702-76e766617a32","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.851713067Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/eb70d53b-bced-473e-b702-76e766617a32","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.852450726Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.852516478Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.852559169Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.85259946Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.857345803Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.866782698Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.982272431Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"uri":"/crucible/0/regions","method":"POST","req_id":"042db6f1-9e51-4064-8b65-e20eea840ed4","remote_addr":"[fd00:1122:3344:101::a]:63558","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.151226648Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"uri":"/crucible/0/regions/eb70d53b-bced-473e-b702-76e766617a32","method":"GET","req_id":"bfc95d85-9a9f-4a5a-875e-fd10ae6d92cd","remote_addr":"[fd00:1122:3344:101::a]:63558","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":2035,"response_code":"200"}
{"msg":"path is \"/data/regions/eb70d53b-bced-473e-b702-76e766617a32\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.153540778Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"datafile"}
{"msg":"dataset is oxp_034b6895-4501-4234-bb8e-9c97c0521fb0/crucible/regions/eb70d53b-bced-473e-b702-76e766617a32","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.165484807Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"datafile"}
{"msg":"region eb70d53b-bced-473e-b702-76e766617a32 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.176029751Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.176182035Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.176240446Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"uri":"/crucible/0/regions/eb70d53b-bced-473e-b702-76e766617a32","method":"DELETE","req_id":"94fd747c-be53-49f4-b6b8-b53d2de3c18d","remote_addr":"[fd00:1122:3344:101::a]:63558","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":22722,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-eb70d53b-bced-473e-b702-76e766617a32 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.177309754Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.180083546Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"uri":"/crucible/0/regions/eb70d53b-bced-473e-b702-76e766617a32","method":"GET","req_id":"43f6d887-e354-4392-9aeb-b26da09e3279","remote_addr":"[fd00:1122:3344:101::a]:63558","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":36,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.181880762Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_034b6895-4501-4234-bb8e-9c97c0521fb0/crucible/regions/eb70d53b-bced-473e-b702-76e766617a32\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.192813846Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"region":"eb70d53b-bced-473e-b702-76e766617a32","component":"worker"}
{"msg":"region eb70d53b-bced-473e-b702-76e766617a32 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.267694467Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.273164368Z","hostname":"oxz_crucible_8ffa8995-7954-461b-b323-57d3138ce992","pid":22736,"uri":"/crucible/0/regions/eb70d53b-bced-473e-b702-76e766617a32","method":"GET","req_id":"70e723a5-07ba-4b76-9ff0-3f7166a3d694","remote_addr":"[fd00:1122:3344:101::a]:63558","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":65,"response_code":"200"}