[ May  8 21:58:25 Enabled. ]
[ May  8 21:58:25 Rereading configuration. ]
[ May  8 21:58:28 Rereading configuration. ]
[ May  8 21:58:31 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_919c09e7-2acf-4d38-a80c-d10b0d488f35/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_919c09e7-2acf-4d38-a80c-d10b0d488f35/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:58:31.679949746Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:58:31.683385255Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:58:31.683397414Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:58:31.701317364Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:58:31.772699887Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:27.993137306Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55699"}
{"msg":"region 491f684d-ae0d-47dd-bd73-0fe2f980c709 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.015480665Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.016433016Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"uri":"/crucible/0/regions","method":"POST","req_id":"1499bf8b-97b5-4df9-8ca2-c1e206c0b41d","remote_addr":"[fd00:1122:3344:101::a]:55699","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":12180,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.017076062Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_919c09e7-2acf-4d38-a80c-d10b0d488f35/crucible/regions/491f684d-ae0d-47dd-bd73-0fe2f980c709","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.027967749Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_919c09e7-2acf-4d38-a80c-d10b0d488f35/crucible/regions/491f684d-ae0d-47dd-bd73-0fe2f980c709","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.028003639Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"491f684d-ae0d-47dd-bd73-0fe2f980c709\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/491f684d-ae0d-47dd-bd73-0fe2f980c709\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.084807447Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"region":"491f684d-ae0d-47dd-bd73-0fe2f980c709","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"491f684d-ae0d-47dd-bd73-0fe2f980c709\" \"--data\" \"/data/regions/491f684d-ae0d-47dd-bd73-0fe2f980c709\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.086860689Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"region":"491f684d-ae0d-47dd-bd73-0fe2f980c709","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.212556983Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"region":"491f684d-ae0d-47dd-bd73-0fe2f980c709","component":"worker"}
{"msg":"region 491f684d-ae0d-47dd-bd73-0fe2f980c709 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.212606794Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.212711785Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-491f684d-ae0d-47dd-bd73-0fe2f980c709","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.213117689Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-491f684d-ae0d-47dd-bd73-0fe2f980c709","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.220897772Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.237004605Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-491f684d-ae0d-47dd-bd73-0fe2f980c709","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.24029842Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/491f684d-ae0d-47dd-bd73-0fe2f980c709","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.241063468Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.24124438Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.241294441Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.241336641Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.245090731Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.250451789Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:28.29548843Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"uri":"/crucible/0/regions","method":"POST","req_id":"46657dd6-0de9-44bc-87d8-4168634337e2","remote_addr":"[fd00:1122:3344:101::a]:55699","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:29.37630039Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"uri":"/crucible/0/regions/491f684d-ae0d-47dd-bd73-0fe2f980c709","method":"GET","req_id":"54dd03b8-1472-46e4-b51a-747939787b87","remote_addr":"[fd00:1122:3344:101::a]:55699","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"path is \"/data/regions/491f684d-ae0d-47dd-bd73-0fe2f980c709\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:29.380178162Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"datafile"}
{"msg":"dataset is oxp_919c09e7-2acf-4d38-a80c-d10b0d488f35/crucible/regions/491f684d-ae0d-47dd-bd73-0fe2f980c709","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:29.391955258Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"datafile"}
{"msg":"region 491f684d-ae0d-47dd-bd73-0fe2f980c709 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:29.402858155Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:29.403029547Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:29.403059087Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"uri":"/crucible/0/regions/491f684d-ae0d-47dd-bd73-0fe2f980c709","method":"DELETE","req_id":"5ad612fb-c075-4a41-bc69-e83cd306e8f2","remote_addr":"[fd00:1122:3344:101::a]:55699","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":22924,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-491f684d-ae0d-47dd-bd73-0fe2f980c709 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:29.40428753Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:29.406681195Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"uri":"/crucible/0/regions/491f684d-ae0d-47dd-bd73-0fe2f980c709","method":"GET","req_id":"bf7b6c15-0e3a-419a-9dec-46f22139bf0c","remote_addr":"[fd00:1122:3344:101::a]:55699","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":31,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:29.4089585Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_919c09e7-2acf-4d38-a80c-d10b0d488f35/crucible/regions/491f684d-ae0d-47dd-bd73-0fe2f980c709\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:29.42208791Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"region":"491f684d-ae0d-47dd-bd73-0fe2f980c709","component":"worker"}
{"msg":"region 491f684d-ae0d-47dd-bd73-0fe2f980c709 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:29.49687349Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:29.53800766Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"uri":"/crucible/0/regions/491f684d-ae0d-47dd-bd73-0fe2f980c709","method":"GET","req_id":"2a21a3d5-8623-4ae4-9447-9cd83c38924a","remote_addr":"[fd00:1122:3344:101::a]:55699","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":61,"response_code":"200"}
{"msg":"region 20e18ce3-e7e1-4e9d-b507-f56ca6b7b307 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:31.800766923Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:31.800950615Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"uri":"/crucible/0/regions","method":"POST","req_id":"0293be9f-0568-4a9c-991f-bf4f4a398ca5","remote_addr":"[fd00:1122:3344:101::a]:55699","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":255,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:31.800973695Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_919c09e7-2acf-4d38-a80c-d10b0d488f35/crucible/regions/20e18ce3-e7e1-4e9d-b507-f56ca6b7b307","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:31.810207444Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_919c09e7-2acf-4d38-a80c-d10b0d488f35/crucible/regions/20e18ce3-e7e1-4e9d-b507-f56ca6b7b307","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:31.810239294Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"20e18ce3-e7e1-4e9d-b507-f56ca6b7b307\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/20e18ce3-e7e1-4e9d-b507-f56ca6b7b307\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:31.897529598Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"region":"20e18ce3-e7e1-4e9d-b507-f56ca6b7b307","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"20e18ce3-e7e1-4e9d-b507-f56ca6b7b307\" \"--data\" \"/data/regions/20e18ce3-e7e1-4e9d-b507-f56ca6b7b307\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:31.899856162Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"region":"20e18ce3-e7e1-4e9d-b507-f56ca6b7b307","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:31.995105711Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"region":"20e18ce3-e7e1-4e9d-b507-f56ca6b7b307","component":"worker"}
{"msg":"region 20e18ce3-e7e1-4e9d-b507-f56ca6b7b307 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:31.995153222Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:31.995268233Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-20e18ce3-e7e1-4e9d-b507-f56ca6b7b307","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:31.996756459Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-20e18ce3-e7e1-4e9d-b507-f56ca6b7b307","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:31.99963254Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:32.007572005Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-20e18ce3-e7e1-4e9d-b507-f56ca6b7b307","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:32.011988562Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/20e18ce3-e7e1-4e9d-b507-f56ca6b7b307","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:32.012276885Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:32.012339126Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:32.012403476Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:32.012473547Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:32.018704074Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:32.025631968Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-08T21:59:32.14645376Z","hostname":"oxz_crucible_09f682cb-4cec-4186-9546-65d7410586be","pid":22736,"uri":"/crucible/0/regions","method":"POST","req_id":"8f85aa2a-745e-4811-adb0-3ce9186a7109","remote_addr":"[fd00:1122:3344:101::a]:55699","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":59,"response_code":"200"}