[ Feb 25 04:21:09 Enabled. ]
[ Feb 25 04:21:09 Rereading configuration. ]
[ Feb 25 04:21:12 Rereading configuration. ]
[ Feb 25 04:21:16 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_dc8b73cb-89ef-41a6-8fed-906219a5be24/crucible -l [fd00:1122:3344:101::12]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_dc8b73cb-89ef-41a6-8fed-906219a5be24/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:21:16.784767725Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716}
{"msg":"listen IP: [fd00:1122:3344:101::12]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:21:16.78783141Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:21:16.78783915Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:21:16.807063695Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:21:16.953033909Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.377304061Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:47754"}
{"msg":"region 568f59bd-1ec7-4fb5-92c4-352696bd6012 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.396094618Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.396285318Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"uri":"/crucible/0/regions","method":"POST","req_id":"147a5b9c-3a13-45b2-8efd-40825a07803f","remote_addr":"[fd00:1122:3344:101::c]:47754","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":7140,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.399281277Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_dc8b73cb-89ef-41a6-8fed-906219a5be24/crucible/regions/568f59bd-1ec7-4fb5-92c4-352696bd6012","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.40892186Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_dc8b73cb-89ef-41a6-8fed-906219a5be24/crucible/regions/568f59bd-1ec7-4fb5-92c4-352696bd6012","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.408955071Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"568f59bd-1ec7-4fb5-92c4-352696bd6012\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/568f59bd-1ec7-4fb5-92c4-352696bd6012\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.505468271Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"region":"568f59bd-1ec7-4fb5-92c4-352696bd6012","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"568f59bd-1ec7-4fb5-92c4-352696bd6012\" \"--data\" \"/data/regions/568f59bd-1ec7-4fb5-92c4-352696bd6012\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.505529302Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"region":"568f59bd-1ec7-4fb5-92c4-352696bd6012","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.6187142Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"region":"568f59bd-1ec7-4fb5-92c4-352696bd6012","component":"worker"}
{"msg":"region 568f59bd-1ec7-4fb5-92c4-352696bd6012 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.618781361Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.618839422Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-568f59bd-1ec7-4fb5-92c4-352696bd6012","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.619262568Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-568f59bd-1ec7-4fb5-92c4-352696bd6012","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.624490776Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.63623761Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-568f59bd-1ec7-4fb5-92c4-352696bd6012","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.639977315Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/568f59bd-1ec7-4fb5-92c4-352696bd6012","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.640676955Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.642769847Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.642827367Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.642865988Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.653172511Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.658595871Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:20.77525591Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"uri":"/crucible/0/regions","method":"POST","req_id":"8976fcc3-395e-415d-a568-7b88c10176ca","remote_addr":"[fd00:1122:3344:101::c]:47754","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:21.831155511Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"uri":"/crucible/0/regions/568f59bd-1ec7-4fb5-92c4-352696bd6012","method":"GET","req_id":"7b61dca2-4cd2-463b-9d0f-90df7e46b750","remote_addr":"[fd00:1122:3344:101::c]:47754","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":53,"response_code":"200"}
{"msg":"path is \"/data/regions/568f59bd-1ec7-4fb5-92c4-352696bd6012\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:21.834424179Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"datafile"}
{"msg":"dataset is oxp_dc8b73cb-89ef-41a6-8fed-906219a5be24/crucible/regions/568f59bd-1ec7-4fb5-92c4-352696bd6012","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:21.84660176Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"datafile"}
{"msg":"region 568f59bd-1ec7-4fb5-92c4-352696bd6012 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:21.856925603Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:21.857060775Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:21.857164647Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"uri":"/crucible/0/regions/568f59bd-1ec7-4fb5-92c4-352696bd6012","method":"DELETE","req_id":"d7fed68d-c5c1-4bfa-9581-3b65825c5456","remote_addr":"[fd00:1122:3344:101::c]:47754","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":22767,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-568f59bd-1ec7-4fb5-92c4-352696bd6012 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:21.858132081Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:21.86075485Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"uri":"/crucible/0/regions/568f59bd-1ec7-4fb5-92c4-352696bd6012","method":"GET","req_id":"ae24dc7d-410f-4c9b-b4ea-e59e4697637d","remote_addr":"[fd00:1122:3344:101::c]:47754","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":28,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:21.864463595Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_dc8b73cb-89ef-41a6-8fed-906219a5be24/crucible/regions/568f59bd-1ec7-4fb5-92c4-352696bd6012\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:21.87629565Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"region":"568f59bd-1ec7-4fb5-92c4-352696bd6012","component":"worker"}
{"msg":"region 568f59bd-1ec7-4fb5-92c4-352696bd6012 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:21.957979071Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:21.968220502Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"uri":"/crucible/0/regions/568f59bd-1ec7-4fb5-92c4-352696bd6012","method":"GET","req_id":"0af76e28-ecfc-4575-baaf-5dbeafa1cb5c","remote_addr":"[fd00:1122:3344:101::c]:47754","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":60,"response_code":"200"}
{"msg":"region 3ceee25e-5b78-4e92-ab0d-9b458bfded6e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.313883411Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.316181375Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.316193745Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"uri":"/crucible/0/regions","method":"POST","req_id":"83a2973c-dc92-4649-a76f-e3064b1a0ceb","remote_addr":"[fd00:1122:3344:101::c]:47754","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":2364,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_dc8b73cb-89ef-41a6-8fed-906219a5be24/crucible/regions/3ceee25e-5b78-4e92-ab0d-9b458bfded6e","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.32460326Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_dc8b73cb-89ef-41a6-8fed-906219a5be24/crucible/regions/3ceee25e-5b78-4e92-ab0d-9b458bfded6e","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.32463069Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3ceee25e-5b78-4e92-ab0d-9b458bfded6e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3ceee25e-5b78-4e92-ab0d-9b458bfded6e\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.383216479Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"region":"3ceee25e-5b78-4e92-ab0d-9b458bfded6e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3ceee25e-5b78-4e92-ab0d-9b458bfded6e\" \"--data\" \"/data/regions/3ceee25e-5b78-4e92-ab0d-9b458bfded6e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.385021966Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"region":"3ceee25e-5b78-4e92-ab0d-9b458bfded6e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.526498563Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"region":"3ceee25e-5b78-4e92-ab0d-9b458bfded6e","component":"worker"}
{"msg":"region 3ceee25e-5b78-4e92-ab0d-9b458bfded6e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.526543893Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.526627664Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3ceee25e-5b78-4e92-ab0d-9b458bfded6e","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.528029005Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3ceee25e-5b78-4e92-ab0d-9b458bfded6e","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.531102751Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.538571932Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3ceee25e-5b78-4e92-ab0d-9b458bfded6e","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.541412974Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3ceee25e-5b78-4e92-ab0d-9b458bfded6e","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.541656457Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.541713518Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.541755029Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.541793999Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.545082988Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.549429522Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T04:22:25.654139795Z","hostname":"oxz_crucible_7036d564-3503-454e-a41b-f2219f4c58cb","pid":22716,"uri":"/crucible/0/regions","method":"POST","req_id":"f2230833-a7fb-4d8f-8561-ccf0be0270d4","remote_addr":"[fd00:1122:3344:101::c]:47754","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":65,"response_code":"200"}