[ Feb 21 07:25:49 Enabled. ]
[ Feb 21 07:25:49 Rereading configuration. ]
[ Feb 21 07:25:51 Rereading configuration. ]
[ Feb 21 07:25:56 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9ce996dc-5236-45ab-a7b4-9b65cd03c049/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_9ce996dc-5236-45ab-a7b4-9b65cd03c049/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:25:56.641862666Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:25:56.645092426Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:25:56.645101646Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:25:56.661623535Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:25:56.845658492Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.218175397Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:34770"}
{"msg":"region 0db0697a-8861-4d6e-a30f-d301bc020987 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.24182428Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.242471273Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"uri":"/crucible/0/regions","method":"POST","req_id":"a8763e41-b121-4535-9cab-664e2f405dff","remote_addr":"[fd00:1122:3344:101::a]:34770","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":9057,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.243711401Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_9ce996dc-5236-45ab-a7b4-9b65cd03c049/crucible/regions/0db0697a-8861-4d6e-a30f-d301bc020987","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.253990263Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9ce996dc-5236-45ab-a7b4-9b65cd03c049/crucible/regions/0db0697a-8861-4d6e-a30f-d301bc020987","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.254019393Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0db0697a-8861-4d6e-a30f-d301bc020987\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0db0697a-8861-4d6e-a30f-d301bc020987\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.321281218Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"region":"0db0697a-8861-4d6e-a30f-d301bc020987","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0db0697a-8861-4d6e-a30f-d301bc020987\" \"--data\" \"/data/regions/0db0697a-8861-4d6e-a30f-d301bc020987\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.323230769Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"region":"0db0697a-8861-4d6e-a30f-d301bc020987","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.438589364Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"region":"0db0697a-8861-4d6e-a30f-d301bc020987","component":"worker"}
{"msg":"region 0db0697a-8861-4d6e-a30f-d301bc020987 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.438637434Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.438719104Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0db0697a-8861-4d6e-a30f-d301bc020987","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.439120207Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0db0697a-8861-4d6e-a30f-d301bc020987","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.44464816Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.456366051Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0db0697a-8861-4d6e-a30f-d301bc020987","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.459530659Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0db0697a-8861-4d6e-a30f-d301bc020987","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.460253944Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.462233336Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.462290246Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.462328596Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.475300564Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.479808541Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.597766211Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"uri":"/crucible/0/regions","method":"POST","req_id":"7f77a9eb-9a12-44ef-b878-af24101ab61b","remote_addr":"[fd00:1122:3344:101::a]:34770","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.851176956Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"uri":"/crucible/0/regions/0db0697a-8861-4d6e-a30f-d301bc020987","method":"GET","req_id":"e4a8ea30-f573-4046-a487-afd66179304b","remote_addr":"[fd00:1122:3344:101::a]:34770","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"path is \"/data/regions/0db0697a-8861-4d6e-a30f-d301bc020987\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.856486398Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"datafile"}
{"msg":"dataset is oxp_9ce996dc-5236-45ab-a7b4-9b65cd03c049/crucible/regions/0db0697a-8861-4d6e-a30f-d301bc020987","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.867406064Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"datafile"}
{"msg":"region 0db0697a-8861-4d6e-a30f-d301bc020987 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.877738946Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.877874427Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-0db0697a-8861-4d6e-a30f-d301bc020987 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.878927173Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.879898839Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"uri":"/crucible/0/regions/0db0697a-8861-4d6e-a30f-d301bc020987","method":"DELETE","req_id":"705075b1-b397-4bff-b07f-2809bff4f859","remote_addr":"[fd00:1122:3344:101::a]:34770","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":23440,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.882965097Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"uri":"/crucible/0/regions/0db0697a-8861-4d6e-a30f-d301bc020987","method":"GET","req_id":"8d66ea05-d5a4-4190-b7e2-5d2ac8494562","remote_addr":"[fd00:1122:3344:101::a]:34770","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":33,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.88352295Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9ce996dc-5236-45ab-a7b4-9b65cd03c049/crucible/regions/0db0697a-8861-4d6e-a30f-d301bc020987\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.894941109Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"region":"0db0697a-8861-4d6e-a30f-d301bc020987","component":"worker"}
{"msg":"region 0db0697a-8861-4d6e-a30f-d301bc020987 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.953518472Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-21T07:26:50.990161052Z","hostname":"oxz_crucible_31b71997-98d2-403b-b6e5-fa5e313b6a48","pid":22943,"uri":"/crucible/0/regions/0db0697a-8861-4d6e-a30f-d301bc020987","method":"GET","req_id":"5ea5f844-056c-4403-9ffb-2383459b04b0","remote_addr":"[fd00:1122:3344:101::a]:34770","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":69,"response_code":"200"}