[ May 22 21:17:24 Enabled. ]
[ May 22 21:17:24 Rereading configuration. ]
[ May 22 21:17:26 Rereading configuration. ]
[ May 22 21:17:29 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_c0db3660-af9b-47a3-b4e0-f5b172db3f55/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_c0db3660-af9b-47a3-b4e0-f5b172db3f55/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:29.940268818Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:29.943242543Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:29.943250873Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:29.956943067Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:30.034184986Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.202400677Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:61398"}
{"msg":"region a8ffce74-77d3-49af-ba5c-e99858a2dfe3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.218967594Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.221836746Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"uri":"/crucible/0/regions","method":"POST","req_id":"47436e58-b617-4bc6-b20b-bdb83b4b43c1","remote_addr":"[fd00:1122:3344:101::c]:61398","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":14290,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.224840821Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_c0db3660-af9b-47a3-b4e0-f5b172db3f55/crucible/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.234081319Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_c0db3660-af9b-47a3-b4e0-f5b172db3f55/crucible/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.234114289Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a8ffce74-77d3-49af-ba5c-e99858a2dfe3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.295165588Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"region":"a8ffce74-77d3-49af-ba5c-e99858a2dfe3","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a8ffce74-77d3-49af-ba5c-e99858a2dfe3\" \"--data\" \"/data/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.295218518Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"region":"a8ffce74-77d3-49af-ba5c-e99858a2dfe3","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.439427315Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"uri":"/crucible/0/regions","method":"POST","req_id":"7cb872fd-475f-4cfd-aab9-9a9b2b899ee1","remote_addr":"[fd00:1122:3344:101::c]:61398","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.602613153Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"region":"a8ffce74-77d3-49af-ba5c-e99858a2dfe3","component":"worker"}
{"msg":"region a8ffce74-77d3-49af-ba5c-e99858a2dfe3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.602659044Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.602772135Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a8ffce74-77d3-49af-ba5c-e99858a2dfe3","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.603219812Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a8ffce74-77d3-49af-ba5c-e99858a2dfe3","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.609857611Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.627351541Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a8ffce74-77d3-49af-ba5c-e99858a2dfe3","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.632073222Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.632817153Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.632897874Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.632943295Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.632982915Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.637583464Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.643780376Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:17:58.797178859Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"uri":"/crucible/0/regions","method":"POST","req_id":"ce17d726-9d0e-4f0e-b6ea-0a10704a3d5e","remote_addr":"[fd00:1122:3344:101::c]:61398","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":46,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.654507481Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:42072"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.658036464Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"uri":"/crucible/0/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3","method":"GET","req_id":"247b61fb-df09-4226-904a-35af45d6bdf5","remote_addr":"[fd00:1122:3344:101::c]:42072","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":3479,"response_code":"200"}
{"msg":"path is \"/data/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.660708284Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"datafile"}
{"msg":"dataset is oxp_c0db3660-af9b-47a3-b4e0-f5b172db3f55/crucible/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.670207525Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"datafile"}
{"msg":"snapshot is oxp_c0db3660-af9b-47a3-b4e0-f5b172db3f55/crucible/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3@da1a9ac6-2a61-425b-b61f-ad2b80183749","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.678284425Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_c0db3660-af9b-47a3-b4e0-f5b172db3f55/crucible/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3\", \"da1a9ac6-2a61-425b-b61f-ad2b80183749\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.678313135Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.688580328Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"uri":"/crucible/0/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3/snapshots/da1a9ac6-2a61-425b-b61f-ad2b80183749","method":"GET","req_id":"8668d903-144b-4337-a60d-0d307c17b5bd","remote_addr":"[fd00:1122:3344:101::c]:42072","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27874,"response_code":"200"}
{"msg":"path is \"/data/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.694144571Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"datafile"}
{"msg":"dataset is oxp_c0db3660-af9b-47a3-b4e0-f5b172db3f55/crucible/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.703837095Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"datafile"}
{"msg":"snapshot is oxp_c0db3660-af9b-47a3-b4e0-f5b172db3f55/crucible/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3@da1a9ac6-2a61-425b-b61f-ad2b80183749","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.712072188Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_c0db3660-af9b-47a3-b4e0-f5b172db3f55/crucible/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3\", \"da1a9ac6-2a61-425b-b61f-ad2b80183749\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.712117189Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot a8ffce74-77d3-49af-ba5c-e99858a2dfe3-da1a9ac6-2a61-425b-b61f-ad2b80183749 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.721314255Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"datafile"}
{"msg":"applying SMF actions for region a8ffce74-77d3-49af-ba5c-e99858a2dfe3 running snapshot da1a9ac6-2a61-425b-b61f-ad2b80183749 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.721936495Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.721959235Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"uri":"/crucible/0/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3/snapshots/da1a9ac6-2a61-425b-b61f-ad2b80183749/run","method":"POST","req_id":"a29acaf7-b6fd-4b2e-b842-eaedba0b7179","remote_addr":"[fd00:1122:3344:101::c]:42072","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27843,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-a8ffce74-77d3-49af-ba5c-e99858a2dfe3-da1a9ac6-2a61-425b-b61f-ad2b80183749","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.737805151Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-a8ffce74-77d3-49af-ba5c-e99858a2dfe3-da1a9ac6-2a61-425b-b61f-ad2b80183749","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.745182831Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-a8ffce74-77d3-49af-ba5c-e99858a2dfe3-da1a9ac6-2a61-425b-b61f-ad2b80183749","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.75725944Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3/.zfs/snapshot/da1a9ac6-2a61-425b-b61f-ad2b80183749","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.757462963Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.757510954Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.757550635Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.757588205Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.757633956Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.761183089Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.76529211Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"worker"}
{"msg":"region a8ffce74-77d3-49af-ba5c-e99858a2dfe3 running snapshot da1a9ac6-2a61-425b-b61f-ad2b80183749 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.76530189Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"datafile"}
{"msg":"path is \"/data/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.955266727Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"datafile"}
{"msg":"dataset is oxp_c0db3660-af9b-47a3-b4e0-f5b172db3f55/crucible/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.964012277Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"datafile"}
{"msg":"snapshot is oxp_c0db3660-af9b-47a3-b4e0-f5b172db3f55/crucible/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3@da1a9ac6-2a61-425b-b61f-ad2b80183749","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.972390762Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_c0db3660-af9b-47a3-b4e0-f5b172db3f55/crucible/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3\", \"da1a9ac6-2a61-425b-b61f-ad2b80183749\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.972425363Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-05-22T21:28:24.980069006Z","hostname":"oxz_crucible_a75bc1b6-5004-4daa-89da-05a3e67295e3","pid":27245,"uri":"/crucible/0/regions/a8ffce74-77d3-49af-ba5c-e99858a2dfe3/snapshots/da1a9ac6-2a61-425b-b61f-ad2b80183749/run","method":"POST","req_id":"27469765-4a28-4aa3-a2c8-983a4da1bf96","remote_addr":"[fd00:1122:3344:101::c]:42072","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":24808,"response_code":"200"}