[ Mar 12 22:35:18 Enabled. ]
[ Mar 12 22:35:18 Rereading configuration. ]
[ Mar 12 22:35:21 Rereading configuration. ]
[ Mar 12 22:35:26 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7b5f136c-ad25-4e92-9622-16cb03d569ab/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_7b5f136c-ad25-4e92-9622-16cb03d569ab/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:35:26.124883519Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:35:26.127903588Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:35:26.127912408Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:35:26.14380212Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:35:26.373342989Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.593010972Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39881"}
{"msg":"region 20e88223-a6b5-41a3-ba67-5edd1257e543 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.620494295Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.622040875Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.625309409Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"uri":"/crucible/0/regions","method":"POST","req_id":"118adcf8-d4ea-41bd-bcca-32c5c6472e75","remote_addr":"[fd00:1122:3344:101::a]:39881","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":15934,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_7b5f136c-ad25-4e92-9622-16cb03d569ab/crucible/regions/20e88223-a6b5-41a3-ba67-5edd1257e543","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.630695579Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_7b5f136c-ad25-4e92-9622-16cb03d569ab/crucible/regions/20e88223-a6b5-41a3-ba67-5edd1257e543","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.6307159Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"20e88223-a6b5-41a3-ba67-5edd1257e543\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/20e88223-a6b5-41a3-ba67-5edd1257e543\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.689421791Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"region":"20e88223-a6b5-41a3-ba67-5edd1257e543","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"20e88223-a6b5-41a3-ba67-5edd1257e543\" \"--data\" \"/data/regions/20e88223-a6b5-41a3-ba67-5edd1257e543\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.689496543Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"region":"20e88223-a6b5-41a3-ba67-5edd1257e543","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.813671371Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"region":"20e88223-a6b5-41a3-ba67-5edd1257e543","component":"worker"}
{"msg":"region 20e88223-a6b5-41a3-ba67-5edd1257e543 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.813729673Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.813798905Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-20e88223-a6b5-41a3-ba67-5edd1257e543","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.814248976Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-20e88223-a6b5-41a3-ba67-5edd1257e543","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.823169328Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.832473409Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-20e88223-a6b5-41a3-ba67-5edd1257e543","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.835387684Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/20e88223-a6b5-41a3-ba67-5edd1257e543","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.836107383Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.836170824Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.836213575Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.836254296Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.840360983Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.850691381Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:14.946515484Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"uri":"/crucible/0/regions","method":"POST","req_id":"cd9e326c-77d8-4722-864b-1309372c045c","remote_addr":"[fd00:1122:3344:101::a]:39881","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":58,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.150625792Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"uri":"/crucible/0/regions/20e88223-a6b5-41a3-ba67-5edd1257e543","method":"GET","req_id":"acd63fe7-e24f-4a72-bf06-ce7abfb5be71","remote_addr":"[fd00:1122:3344:101::a]:39881","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":2040,"response_code":"200"}
{"msg":"path is \"/data/regions/20e88223-a6b5-41a3-ba67-5edd1257e543\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.153811455Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"datafile"}
{"msg":"dataset is oxp_7b5f136c-ad25-4e92-9622-16cb03d569ab/crucible/regions/20e88223-a6b5-41a3-ba67-5edd1257e543","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.166387761Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"datafile"}
{"msg":"region 20e88223-a6b5-41a3-ba67-5edd1257e543 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.176930934Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.177084728Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.17714598Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"uri":"/crucible/0/regions/20e88223-a6b5-41a3-ba67-5edd1257e543","method":"DELETE","req_id":"b56cb04f-ca4d-4446-bde6-a6e9e0414772","remote_addr":"[fd00:1122:3344:101::a]:39881","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":23364,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-20e88223-a6b5-41a3-ba67-5edd1257e543 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.17832986Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.180535337Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"uri":"/crucible/0/regions/20e88223-a6b5-41a3-ba67-5edd1257e543","method":"GET","req_id":"49de9f1e-6ffd-43e6-b7cc-1b5b852aea3b","remote_addr":"[fd00:1122:3344:101::a]:39881","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":33,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.182328774Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_7b5f136c-ad25-4e92-9622-16cb03d569ab/crucible/regions/20e88223-a6b5-41a3-ba67-5edd1257e543\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.194285954Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"region":"20e88223-a6b5-41a3-ba67-5edd1257e543","component":"worker"}
{"msg":"region 20e88223-a6b5-41a3-ba67-5edd1257e543 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.267414179Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-12T22:36:16.284174384Z","hostname":"oxz_crucible_4c7dabdc-fbca-47de-98a3-f4fc1a449e3d","pid":22993,"uri":"/crucible/0/regions/20e88223-a6b5-41a3-ba67-5edd1257e543","method":"GET","req_id":"cabae3c0-310b-4ef3-9376-91e9d7b034e2","remote_addr":"[fd00:1122:3344:101::a]:39881","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":68,"response_code":"200"}