[ Feb 28 06:45:55 Enabled. ]
[ Feb 28 06:45:55 Rereading configuration. ]
[ Feb 28 06:45:58 Rereading configuration. ]
[ Feb 28 06:46:01 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_3430e022-7133-4293-ac77-634fd73d7c3b/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_3430e022-7133-4293-ac77-634fd73d7c3b/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:46:01.190072035Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:46:01.193145551Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:46:01.193156192Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:46:01.209531368Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:46:01.3290912Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:23.755325116Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64724"}
{"msg":"region 3163bff5-f19d-4014-9379-12a641e7ff26 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:23.822945496Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:23.827860935Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"uri":"/crucible/0/regions","method":"POST","req_id":"ff18be08-ef4c-4f6b-bfda-a7535c49e0fb","remote_addr":"[fd00:1122:3344:101::a]:64724","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":27464,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:23.831765402Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_3430e022-7133-4293-ac77-634fd73d7c3b/crucible/regions/3163bff5-f19d-4014-9379-12a641e7ff26","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:23.851525058Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_3430e022-7133-4293-ac77-634fd73d7c3b/crucible/regions/3163bff5-f19d-4014-9379-12a641e7ff26","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:23.851574399Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"3163bff5-f19d-4014-9379-12a641e7ff26\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3163bff5-f19d-4014-9379-12a641e7ff26\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:23.946441095Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"region":"3163bff5-f19d-4014-9379-12a641e7ff26","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3163bff5-f19d-4014-9379-12a641e7ff26\" \"--data\" \"/data/regions/3163bff5-f19d-4014-9379-12a641e7ff26\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:23.949555163Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"region":"3163bff5-f19d-4014-9379-12a641e7ff26","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.155452249Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"uri":"/crucible/0/regions","method":"POST","req_id":"a35f34d3-0b12-4f5d-917f-dd182fd42c81","remote_addr":"[fd00:1122:3344:101::a]:64724","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.29742157Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"region":"3163bff5-f19d-4014-9379-12a641e7ff26","component":"worker"}
{"msg":"region 3163bff5-f19d-4014-9379-12a641e7ff26 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.297471851Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.297571352Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-3163bff5-f19d-4014-9379-12a641e7ff26","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.298023837Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3163bff5-f19d-4014-9379-12a641e7ff26","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.309062619Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.328176919Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3163bff5-f19d-4014-9379-12a641e7ff26","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.331007413Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3163bff5-f19d-4014-9379-12a641e7ff26","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.331736851Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.331801232Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.331847323Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.331888783Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.336177364Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.360199022Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:24.877519428Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"uri":"/crucible/0/regions","method":"POST","req_id":"25a0ac91-d5b3-4032-9a75-1ffd4387746c","remote_addr":"[fd00:1122:3344:101::a]:64724","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":56,"response_code":"200"}
{"msg":"region 57b69746-1c97-4ca1-a3a8-d5a0aafe537e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.392103487Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.392281399Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.392294839Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"uri":"/crucible/0/regions","method":"POST","req_id":"537c8809-114d-4a7b-b6eb-ef050e12c880","remote_addr":"[fd00:1122:3344:101::a]:64724","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":221,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_3430e022-7133-4293-ac77-634fd73d7c3b/crucible/regions/57b69746-1c97-4ca1-a3a8-d5a0aafe537e","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.402186688Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_3430e022-7133-4293-ac77-634fd73d7c3b/crucible/regions/57b69746-1c97-4ca1-a3a8-d5a0aafe537e","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.402219638Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"57b69746-1c97-4ca1-a3a8-d5a0aafe537e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/57b69746-1c97-4ca1-a3a8-d5a0aafe537e\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.463079347Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"region":"57b69746-1c97-4ca1-a3a8-d5a0aafe537e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"57b69746-1c97-4ca1-a3a8-d5a0aafe537e\" \"--data\" \"/data/regions/57b69746-1c97-4ca1-a3a8-d5a0aafe537e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.46495687Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"region":"57b69746-1c97-4ca1-a3a8-d5a0aafe537e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.537053583Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"region":"57b69746-1c97-4ca1-a3a8-d5a0aafe537e","component":"worker"}
{"msg":"region 57b69746-1c97-4ca1-a3a8-d5a0aafe537e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.537086014Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.537203205Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-57b69746-1c97-4ca1-a3a8-d5a0aafe537e","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.538647733Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-57b69746-1c97-4ca1-a3a8-d5a0aafe537e","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.541192993Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.550679957Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-57b69746-1c97-4ca1-a3a8-d5a0aafe537e","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.554400041Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/57b69746-1c97-4ca1-a3a8-d5a0aafe537e","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.554678785Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.554735805Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.554778706Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.554817546Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.557888333Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.562240085Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:30.575010988Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"uri":"/crucible/0/regions","method":"POST","req_id":"2315bde1-a821-4077-83c7-ffbaa2d84afa","remote_addr":"[fd00:1122:3344:101::a]:64724","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:55.321118677Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"uri":"/crucible/0/regions/57b69746-1c97-4ca1-a3a8-d5a0aafe537e","method":"GET","req_id":"0bc5c512-9fb1-47e8-80f6-43e7f81141bc","remote_addr":"[fd00:1122:3344:101::a]:64724","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":2702,"response_code":"200"}
{"msg":"path is \"/data/regions/57b69746-1c97-4ca1-a3a8-d5a0aafe537e\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:55.32380807Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"datafile"}
{"msg":"dataset is oxp_3430e022-7133-4293-ac77-634fd73d7c3b/crucible/regions/57b69746-1c97-4ca1-a3a8-d5a0aafe537e","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:55.335746062Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"datafile"}
{"msg":"region 57b69746-1c97-4ca1-a3a8-d5a0aafe537e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:55.345671391Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:55.345870734Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:55.345927784Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"uri":"/crucible/0/regions/57b69746-1c97-4ca1-a3a8-d5a0aafe537e","method":"DELETE","req_id":"4f065199-70e6-4daa-bb0c-a80f5f1c75a6","remote_addr":"[fd00:1122:3344:101::a]:64724","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":22146,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-57b69746-1c97-4ca1-a3a8-d5a0aafe537e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:55.346952357Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:55.34892117Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"uri":"/crucible/0/regions/57b69746-1c97-4ca1-a3a8-d5a0aafe537e","method":"GET","req_id":"82464755-3bb3-4d48-bec5-1e25a843509b","remote_addr":"[fd00:1122:3344:101::a]:64724","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:55.351522842Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_3430e022-7133-4293-ac77-634fd73d7c3b/crucible/regions/57b69746-1c97-4ca1-a3a8-d5a0aafe537e\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:55.360858223Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"region":"57b69746-1c97-4ca1-a3a8-d5a0aafe537e","component":"worker"}
{"msg":"region 57b69746-1c97-4ca1-a3a8-d5a0aafe537e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:55.418672816Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-28T06:57:55.488708785Z","hostname":"oxz_crucible_17130402-2073-40d7-b72b-01697b9cffc7","pid":27295,"uri":"/crucible/0/regions/57b69746-1c97-4ca1-a3a8-d5a0aafe537e","method":"GET","req_id":"edc03d73-f6ec-436e-a6b0-c1f3dba2739a","remote_addr":"[fd00:1122:3344:101::a]:64724","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":54,"response_code":"200"}