[ Nov 19 05:52:54 Enabled. ]
[ Nov 19 05:52:54 Rereading configuration. ]
[ Nov 19 05:52:59 Rereading configuration. ]
[ Nov 19 05:53:06 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_35a1f44b-de46-41ee-ad1e-96313ecc15b9/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_35a1f44b-de46-41ee-ad1e-96313ecc15b9/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:06.432107334Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:06.434896563Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:06.434909923Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:06.449296412Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:06.574050387Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.276713882Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:57632"}
{"msg":"region 94873457-8ed2-4dd6-bed1-2273ee8bec3f state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.304737033Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.306448961Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.31019984Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"uri":"/crucible/0/regions","method":"POST","req_id":"33921727-6c5e-470b-9a44-df4e3c1b931c","remote_addr":"[fd00:1122:3344:101::b]:57632","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":13363,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_35a1f44b-de46-41ee-ad1e-96313ecc15b9/crucible/regions/94873457-8ed2-4dd6-bed1-2273ee8bec3f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.316251732Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_35a1f44b-de46-41ee-ad1e-96313ecc15b9/crucible/regions/94873457-8ed2-4dd6-bed1-2273ee8bec3f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.316272833Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"94873457-8ed2-4dd6-bed1-2273ee8bec3f\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/94873457-8ed2-4dd6-bed1-2273ee8bec3f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.373704179Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"region":"94873457-8ed2-4dd6-bed1-2273ee8bec3f","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"94873457-8ed2-4dd6-bed1-2273ee8bec3f\" \"--data\" \"/data/regions/94873457-8ed2-4dd6-bed1-2273ee8bec3f\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.37574439Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"region":"94873457-8ed2-4dd6-bed1-2273ee8bec3f","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.48652787Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"region":"94873457-8ed2-4dd6-bed1-2273ee8bec3f","component":"worker"}
{"msg":"region 94873457-8ed2-4dd6-bed1-2273ee8bec3f state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.48657208Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.486657771Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-94873457-8ed2-4dd6-bed1-2273ee8bec3f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.487062955Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-94873457-8ed2-4dd6-bed1-2273ee8bec3f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.491630432Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.507213914Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-94873457-8ed2-4dd6-bed1-2273ee8bec3f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.510517408Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/94873457-8ed2-4dd6-bed1-2273ee8bec3f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.511439158Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.513397928Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.513452279Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.513492549Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.523470933Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.528221012Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:40.563143851Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"uri":"/crucible/0/regions","method":"POST","req_id":"aebf57fe-d1ef-4ac8-ad05-bd4cb125b428","remote_addr":"[fd00:1122:3344:101::b]:57632","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:41.287412533Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"uri":"/crucible/0/regions/94873457-8ed2-4dd6-bed1-2273ee8bec3f","method":"GET","req_id":"f472e052-2e4e-439d-b058-ab2efd3c849b","remote_addr":"[fd00:1122:3344:101::b]:57632","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"path is \"/data/regions/94873457-8ed2-4dd6-bed1-2273ee8bec3f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:41.291329524Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"datafile"}
{"msg":"dataset is oxp_35a1f44b-de46-41ee-ad1e-96313ecc15b9/crucible/regions/94873457-8ed2-4dd6-bed1-2273ee8bec3f","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:41.302773862Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"datafile"}
{"msg":"region 94873457-8ed2-4dd6-bed1-2273ee8bec3f state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:41.312765586Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:41.312898417Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-94873457-8ed2-4dd6-bed1-2273ee8bec3f (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:41.313882738Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:41.314821137Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"uri":"/crucible/0/regions/94873457-8ed2-4dd6-bed1-2273ee8bec3f","method":"DELETE","req_id":"51b01733-0bdf-4fb0-90ca-616a2752c2e5","remote_addr":"[fd00:1122:3344:101::b]:57632","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":23516,"response_code":"204"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:41.31797641Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:41.321140033Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"uri":"/crucible/0/regions/94873457-8ed2-4dd6-bed1-2273ee8bec3f","method":"GET","req_id":"be9dfaee-24cd-45e0-9c4e-ce8921191ff0","remote_addr":"[fd00:1122:3344:101::b]:57632","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":630,"response_code":"200"}
{"msg":"deleting zfs dataset \"oxp_35a1f44b-de46-41ee-ad1e-96313ecc15b9/crucible/regions/94873457-8ed2-4dd6-bed1-2273ee8bec3f\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:41.327813492Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"region":"94873457-8ed2-4dd6-bed1-2273ee8bec3f","component":"worker"}
{"msg":"region 94873457-8ed2-4dd6-bed1-2273ee8bec3f state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:41.382682942Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:53:41.458226225Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"uri":"/crucible/0/regions/94873457-8ed2-4dd6-bed1-2273ee8bec3f","method":"GET","req_id":"95499839-ef74-45a9-afbd-e226f144697e","remote_addr":"[fd00:1122:3344:101::b]:57632","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":75,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.311336655Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52792"}
{"msg":"region 69e1cc76-85fc-4d9e-a133-8fdc0f0959ba state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.311579397Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.311773779Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.3118023Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"uri":"/crucible/0/regions","method":"POST","req_id":"818cb438-e7eb-4486-85ee-248b36997665","remote_addr":"[fd00:1122:3344:101::a]:52792","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":226,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_35a1f44b-de46-41ee-ad1e-96313ecc15b9/crucible/regions/69e1cc76-85fc-4d9e-a133-8fdc0f0959ba","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.320952265Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_35a1f44b-de46-41ee-ad1e-96313ecc15b9/crucible/regions/69e1cc76-85fc-4d9e-a133-8fdc0f0959ba","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.320987315Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"69e1cc76-85fc-4d9e-a133-8fdc0f0959ba\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/69e1cc76-85fc-4d9e-a133-8fdc0f0959ba\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.371112275Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"region":"69e1cc76-85fc-4d9e-a133-8fdc0f0959ba","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"69e1cc76-85fc-4d9e-a133-8fdc0f0959ba\" \"--data\" \"/data/regions/69e1cc76-85fc-4d9e-a133-8fdc0f0959ba\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.371168476Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"region":"69e1cc76-85fc-4d9e-a133-8fdc0f0959ba","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.44378618Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"region":"69e1cc76-85fc-4d9e-a133-8fdc0f0959ba","component":"worker"}
{"msg":"region 69e1cc76-85fc-4d9e-a133-8fdc0f0959ba state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.44382327Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.443933951Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-69e1cc76-85fc-4d9e-a133-8fdc0f0959ba","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.445458647Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-69e1cc76-85fc-4d9e-a133-8fdc0f0959ba","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.448776392Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.45732231Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-69e1cc76-85fc-4d9e-a133-8fdc0f0959ba","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.460250651Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/69e1cc76-85fc-4d9e-a133-8fdc0f0959ba","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.460558454Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.460614634Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.460657725Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.460698535Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.464496185Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.473100814Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-19T05:54:17.568380983Z","hostname":"oxz_crucible_9a256def-dc34-4e28-a778-837b612c3e75","pid":23335,"uri":"/crucible/0/regions","method":"POST","req_id":"72fe6c8d-fb65-4abf-8a15-6ffcc36dd0b6","remote_addr":"[fd00:1122:3344:101::a]:52792","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":60,"response_code":"200"}