[ Nov 21 01:16:56 Enabled. ]
[ Nov 21 01:16:56 Rereading configuration. ]
[ Nov 21 01:17:00 Rereading configuration. ]
[ Nov 21 01:17:04 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:04.32987924Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:04.33275328Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:04.332766181Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:04.35047499Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:04.455632942Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.642793462Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:40243"}
{"msg":"region 7eef381c-bce5-4086-88a8-d844a795e28d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.65404705Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.654408135Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"uri":"/crucible/0/regions","method":"POST","req_id":"9bbcc26b-92df-48de-86bf-ef3b90b037a1","remote_addr":"[fd00:1122:3344:101::a]:40243","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":4722,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.655615352Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible/regions/7eef381c-bce5-4086-88a8-d844a795e28d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.665232298Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible/regions/7eef381c-bce5-4086-88a8-d844a795e28d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.665285569Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"7eef381c-bce5-4086-88a8-d844a795e28d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7eef381c-bce5-4086-88a8-d844a795e28d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.730831332Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"region":"7eef381c-bce5-4086-88a8-d844a795e28d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7eef381c-bce5-4086-88a8-d844a795e28d\" \"--data\" \"/data/regions/7eef381c-bce5-4086-88a8-d844a795e28d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.735056912Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"region":"7eef381c-bce5-4086-88a8-d844a795e28d","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.884715851Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"region":"7eef381c-bce5-4086-88a8-d844a795e28d","component":"worker"}
{"msg":"region 7eef381c-bce5-4086-88a8-d844a795e28d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.884754202Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.884861973Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-7eef381c-bce5-4086-88a8-d844a795e28d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.88533949Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7eef381c-bce5-4086-88a8-d844a795e28d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.888629156Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.896533848Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7eef381c-bce5-4086-88a8-d844a795e28d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.900021106Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7eef381c-bce5-4086-88a8-d844a795e28d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.900666066Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.900758297Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.900808258Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.900850968Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.905784548Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.909879196Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:17:21.97334959Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"uri":"/crucible/0/regions","method":"POST","req_id":"540b2f80-d4b5-4a5e-8a93-ddceadcf14f4","remote_addr":"[fd00:1122:3344:101::a]:40243","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.677642435Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:36618"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.685309483Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"uri":"/crucible/0/regions/7eef381c-bce5-4086-88a8-d844a795e28d","method":"GET","req_id":"e872445a-ae0d-4fec-9f7e-789a33f59c75","remote_addr":"[fd00:1122:3344:101::a]:36618","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":7619,"response_code":"200"}
{"msg":"path is \"/data/regions/7eef381c-bce5-4086-88a8-d844a795e28d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.689527412Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"datafile"}
{"msg":"dataset is oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible/regions/7eef381c-bce5-4086-88a8-d844a795e28d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.698737542Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"datafile"}
{"msg":"snapshot is oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible/regions/7eef381c-bce5-4086-88a8-d844a795e28d@8b520afc-9001-47a5-8f88-447d6d5f59d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.70639261Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible/regions/7eef381c-bce5-4086-88a8-d844a795e28d\", \"8b520afc-9001-47a5-8f88-447d6d5f59d7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.70642738Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.716305819Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"uri":"/crucible/0/regions/7eef381c-bce5-4086-88a8-d844a795e28d/snapshots/8b520afc-9001-47a5-8f88-447d6d5f59d7","method":"GET","req_id":"3bebe5f5-d27a-40ac-a513-65d97b897955","remote_addr":"[fd00:1122:3344:101::a]:36618","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26792,"response_code":"200"}
{"msg":"path is \"/data/regions/7eef381c-bce5-4086-88a8-d844a795e28d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.720081062Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"datafile"}
{"msg":"dataset is oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible/regions/7eef381c-bce5-4086-88a8-d844a795e28d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.729331863Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"datafile"}
{"msg":"snapshot is oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible/regions/7eef381c-bce5-4086-88a8-d844a795e28d@8b520afc-9001-47a5-8f88-447d6d5f59d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.736981151Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible/regions/7eef381c-bce5-4086-88a8-d844a795e28d\", \"8b520afc-9001-47a5-8f88-447d6d5f59d7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.737027861Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 7eef381c-bce5-4086-88a8-d844a795e28d-8b520afc-9001-47a5-8f88-447d6d5f59d7 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.745894086Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"datafile"}
{"msg":"applying SMF actions for region 7eef381c-bce5-4086-88a8-d844a795e28d running snapshot 8b520afc-9001-47a5-8f88-447d6d5f59d7 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.746693897Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.746749638Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"uri":"/crucible/0/regions/7eef381c-bce5-4086-88a8-d844a795e28d/snapshots/8b520afc-9001-47a5-8f88-447d6d5f59d7/run","method":"POST","req_id":"ed62a250-4710-4862-b4ee-dce19e231b8f","remote_addr":"[fd00:1122:3344:101::a]:36618","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":26715,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-7eef381c-bce5-4086-88a8-d844a795e28d-8b520afc-9001-47a5-8f88-447d6d5f59d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.752982246Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-7eef381c-bce5-4086-88a8-d844a795e28d-8b520afc-9001-47a5-8f88-447d6d5f59d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.755842966Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-7eef381c-bce5-4086-88a8-d844a795e28d-8b520afc-9001-47a5-8f88-447d6d5f59d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.767227417Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7eef381c-bce5-4086-88a8-d844a795e28d/.zfs/snapshot/8b520afc-9001-47a5-8f88-447d6d5f59d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.76742565Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.76747378Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.767512411Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.767549711Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.767594542Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.772030855Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.776872743Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"worker"}
{"msg":"region 7eef381c-bce5-4086-88a8-d844a795e28d running snapshot 8b520afc-9001-47a5-8f88-447d6d5f59d7 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.776892333Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"datafile"}
{"msg":"path is \"/data/regions/7eef381c-bce5-4086-88a8-d844a795e28d\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.9193058Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"datafile"}
{"msg":"dataset is oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible/regions/7eef381c-bce5-4086-88a8-d844a795e28d","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.928267876Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"datafile"}
{"msg":"snapshot is oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible/regions/7eef381c-bce5-4086-88a8-d844a795e28d@8b520afc-9001-47a5-8f88-447d6d5f59d7","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.936798716Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_db8007cb-fccc-44e3-84e8-02e7abe2a72a/crucible/regions/7eef381c-bce5-4086-88a8-d844a795e28d\", \"8b520afc-9001-47a5-8f88-447d6d5f59d7\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.936841257Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-21T01:27:46.944438474Z","hostname":"oxz_crucible_c605ae15-9986-48f2-8313-f0f8aa776282","pid":26800,"uri":"/crucible/0/regions/7eef381c-bce5-4086-88a8-d844a795e28d/snapshots/8b520afc-9001-47a5-8f88-447d6d5f59d7/run","method":"POST","req_id":"c4b1b2fa-19da-4ffe-990e-cf4ae5d225e0","remote_addr":"[fd00:1122:3344:101::a]:36618","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25187,"response_code":"200"}