[ Aug 11 21:33:22 Enabled. ]
[ Aug 11 21:33:22 Rereading configuration. ]
[ Aug 11 21:33:25 Rereading configuration. ]
[ Aug 11 21:33:30 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_9901ebe7-9339-4383-94e7-6e9b86645f40/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_9901ebe7-9339-4383-94e7-6e9b86645f40/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:33:30.221313012Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:33:30.224275002Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:33:30.224283732Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:33:30.241284677Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:33:30.357911621Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.518407956Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63880"}
{"msg":"region 8870a66a-48a4-4289-b5a9-189d42c2ccd2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.553984002Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.558673571Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"uri":"/crucible/0/regions","method":"POST","req_id":"48182999-628f-4f9a-a11e-353abd710a13","remote_addr":"[fd00:1122:3344:101::a]:63880","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20301,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.56097272Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_9901ebe7-9339-4383-94e7-6e9b86645f40/crucible/regions/8870a66a-48a4-4289-b5a9-189d42c2ccd2","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.570241455Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_9901ebe7-9339-4383-94e7-6e9b86645f40/crucible/regions/8870a66a-48a4-4289-b5a9-189d42c2ccd2","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.570276376Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"8870a66a-48a4-4289-b5a9-189d42c2ccd2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8870a66a-48a4-4289-b5a9-189d42c2ccd2\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.633161609Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"region":"8870a66a-48a4-4289-b5a9-189d42c2ccd2","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8870a66a-48a4-4289-b5a9-189d42c2ccd2\" \"--data\" \"/data/regions/8870a66a-48a4-4289-b5a9-189d42c2ccd2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.63321221Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"region":"8870a66a-48a4-4289-b5a9-189d42c2ccd2","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.762002139Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"uri":"/crucible/0/regions","method":"POST","req_id":"657cbbaa-ec35-47ca-b7a3-310e2a2c0480","remote_addr":"[fd00:1122:3344:101::a]:63880","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":62,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.910993635Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"region":"8870a66a-48a4-4289-b5a9-189d42c2ccd2","component":"worker"}
{"msg":"region 8870a66a-48a4-4289-b5a9-189d42c2ccd2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.911032095Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.911135937Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-8870a66a-48a4-4289-b5a9-189d42c2ccd2","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.911573834Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8870a66a-48a4-4289-b5a9-189d42c2ccd2","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.918678644Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.936794127Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8870a66a-48a4-4289-b5a9-189d42c2ccd2","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.939912259Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8870a66a-48a4-4289-b5a9-189d42c2ccd2","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.940580731Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.940649332Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.940691793Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.940728003Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.943988028Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:31.957208919Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:32.048395937Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"uri":"/crucible/0/regions","method":"POST","req_id":"ad17bc37-844b-49ca-9bb5-bcdb8d21f759","remote_addr":"[fd00:1122:3344:101::a]:63880","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":43,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:32.967317576Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"uri":"/crucible/0/regions/8870a66a-48a4-4289-b5a9-189d42c2ccd2","method":"GET","req_id":"0cc1bb54-4930-41cd-a4c5-88d073ef2161","remote_addr":"[fd00:1122:3344:101::a]:63880","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"path is \"/data/regions/8870a66a-48a4-4289-b5a9-189d42c2ccd2\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:32.969676195Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"datafile"}
{"msg":"dataset is oxp_9901ebe7-9339-4383-94e7-6e9b86645f40/crucible/regions/8870a66a-48a4-4289-b5a9-189d42c2ccd2","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:32.981209558Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"datafile"}
{"msg":"region 8870a66a-48a4-4289-b5a9-189d42c2ccd2 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:32.990903091Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:32.991082604Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:32.991158395Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"uri":"/crucible/0/regions/8870a66a-48a4-4289-b5a9-189d42c2ccd2","method":"DELETE","req_id":"c69155f7-3dc1-4d0c-9d04-70a0f205cff3","remote_addr":"[fd00:1122:3344:101::a]:63880","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":21503,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-8870a66a-48a4-4289-b5a9-189d42c2ccd2 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:32.992263764Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:32.994228036Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"uri":"/crucible/0/regions/8870a66a-48a4-4289-b5a9-189d42c2ccd2","method":"GET","req_id":"08069b56-992f-4234-85d5-ae7f32f393cf","remote_addr":"[fd00:1122:3344:101::a]:63880","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":22,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:32.996284551Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_9901ebe7-9339-4383-94e7-6e9b86645f40/crucible/regions/8870a66a-48a4-4289-b5a9-189d42c2ccd2\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:33.006784367Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"region":"8870a66a-48a4-4289-b5a9-189d42c2ccd2","component":"worker"}
{"msg":"region 8870a66a-48a4-4289-b5a9-189d42c2ccd2 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:33.073585986Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:33.091452876Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"uri":"/crucible/0/regions/8870a66a-48a4-4289-b5a9-189d42c2ccd2","method":"GET","req_id":"fde75555-9e23-4367-8e7d-ff99f169235c","remote_addr":"[fd00:1122:3344:101::a]:63880","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"region 6130c5c3-2a44-498d-89ff-0c6557cc08be state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.273716334Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.273869656Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.273904507Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"uri":"/crucible/0/regions","method":"POST","req_id":"952dea67-f993-4eda-b3ce-23cd5055790d","remote_addr":"[fd00:1122:3344:101::a]:63880","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":215,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_9901ebe7-9339-4383-94e7-6e9b86645f40/crucible/regions/6130c5c3-2a44-498d-89ff-0c6557cc08be","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.282811586Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_9901ebe7-9339-4383-94e7-6e9b86645f40/crucible/regions/6130c5c3-2a44-498d-89ff-0c6557cc08be","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.282834317Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"6130c5c3-2a44-498d-89ff-0c6557cc08be\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/6130c5c3-2a44-498d-89ff-0c6557cc08be\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.332479598Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"region":"6130c5c3-2a44-498d-89ff-0c6557cc08be","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"6130c5c3-2a44-498d-89ff-0c6557cc08be\" \"--data\" \"/data/regions/6130c5c3-2a44-498d-89ff-0c6557cc08be\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.332525469Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"region":"6130c5c3-2a44-498d-89ff-0c6557cc08be","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.415043402Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"region":"6130c5c3-2a44-498d-89ff-0c6557cc08be","component":"worker"}
{"msg":"region 6130c5c3-2a44-498d-89ff-0c6557cc08be state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.415086263Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.415167574Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-6130c5c3-2a44-498d-89ff-0c6557cc08be","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.416440076Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-6130c5c3-2a44-498d-89ff-0c6557cc08be","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.41966271Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.42686753Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-6130c5c3-2a44-498d-89ff-0c6557cc08be","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.430131175Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/6130c5c3-2a44-498d-89ff-0c6557cc08be","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.430344038Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.430390269Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.43042612Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.43045975Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.433799986Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.438001567Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-08-11T21:34:35.632673929Z","hostname":"oxz_crucible_0655882f-546d-4247-8005-0a1521ed2480","pid":23571,"uri":"/crucible/0/regions","method":"POST","req_id":"3cf28797-b61f-44cf-a0a1-fa4ea431e9aa","remote_addr":"[fd00:1122:3344:101::a]:63880","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":55,"response_code":"200"}