[ Jan 14 22:31:23 Enabled. ]
[ Jan 14 22:31:23 Rereading configuration. ]
[ Jan 14 22:31:25 Rereading configuration. ]
[ Jan 14 22:31:29 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_489724fa-785f-4f35-bb0f-34adf6a09482/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_489724fa-785f-4f35-bb0f-34adf6a09482/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:29.115758663Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:29.120714903Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:29.120729603Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:29.134171104Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:29.21163328Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.248885122Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:47937"}
{"msg":"region 296a0247-a1f1-4435-8d1c-2c97083c4080 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.269173214Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.271998548Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.277019848Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"uri":"/crucible/0/regions","method":"POST","req_id":"64a2448a-7296-43c4-b7af-cb26fc1f67df","remote_addr":"[fd00:1122:3344:101::a]:47937","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":11772,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_489724fa-785f-4f35-bb0f-34adf6a09482/crucible/regions/296a0247-a1f1-4435-8d1c-2c97083c4080","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.284164993Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_489724fa-785f-4f35-bb0f-34adf6a09482/crucible/regions/296a0247-a1f1-4435-8d1c-2c97083c4080","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.284212044Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"296a0247-a1f1-4435-8d1c-2c97083c4080\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/296a0247-a1f1-4435-8d1c-2c97083c4080\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.362306908Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"region":"296a0247-a1f1-4435-8d1c-2c97083c4080","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"296a0247-a1f1-4435-8d1c-2c97083c4080\" \"--data\" \"/data/regions/296a0247-a1f1-4435-8d1c-2c97083c4080\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.364247751Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"region":"296a0247-a1f1-4435-8d1c-2c97083c4080","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.545765081Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"uri":"/crucible/0/regions","method":"POST","req_id":"6c1315df-a0b3-49db-8efd-d5d9b5ba89d3","remote_addr":"[fd00:1122:3344:101::a]:47937","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":88,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.564477175Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"region":"296a0247-a1f1-4435-8d1c-2c97083c4080","component":"worker"}
{"msg":"region 296a0247-a1f1-4435-8d1c-2c97083c4080 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.564531756Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.564753699Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-296a0247-a1f1-4435-8d1c-2c97083c4080","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.565138493Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-296a0247-a1f1-4435-8d1c-2c97083c4080","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.570088742Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.579814879Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-296a0247-a1f1-4435-8d1c-2c97083c4080","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.582999457Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/296a0247-a1f1-4435-8d1c-2c97083c4080","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.583756346Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.583821107Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.583858727Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.583897877Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.588434592Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:47.594360862Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:31:48.124351081Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"uri":"/crucible/0/regions","method":"POST","req_id":"1c2d3e57-9140-4d36-ae91-dcec340029b5","remote_addr":"[fd00:1122:3344:101::a]:47937","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":59,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.973863404Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:37670"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.977825661Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"uri":"/crucible/0/regions/296a0247-a1f1-4435-8d1c-2c97083c4080","method":"GET","req_id":"a9f50515-2be0-46b2-a564-030cf53a2eb4","remote_addr":"[fd00:1122:3344:101::a]:37670","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":3915,"response_code":"200"}
{"msg":"path is \"/data/regions/296a0247-a1f1-4435-8d1c-2c97083c4080\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.981530535Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"datafile"}
{"msg":"dataset is oxp_489724fa-785f-4f35-bb0f-34adf6a09482/crucible/regions/296a0247-a1f1-4435-8d1c-2c97083c4080","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.990524063Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"datafile"}
{"msg":"snapshot is oxp_489724fa-785f-4f35-bb0f-34adf6a09482/crucible/regions/296a0247-a1f1-4435-8d1c-2c97083c4080@ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.999262548Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_489724fa-785f-4f35-bb0f-34adf6a09482/crucible/regions/296a0247-a1f1-4435-8d1c-2c97083c4080\", \"ed7e43c3-4205-4e31-b98f-42e7a917cf19\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:40.999294098Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.011289071Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"uri":"/crucible/0/regions/296a0247-a1f1-4435-8d1c-2c97083c4080/snapshots/ed7e43c3-4205-4e31-b98f-42e7a917cf19","method":"GET","req_id":"b6a4bce9-5af0-4160-ba19-0012a34035d7","remote_addr":"[fd00:1122:3344:101::a]:37670","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":29765,"response_code":"200"}
{"msg":"path is \"/data/regions/296a0247-a1f1-4435-8d1c-2c97083c4080\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.014988086Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"datafile"}
{"msg":"dataset is oxp_489724fa-785f-4f35-bb0f-34adf6a09482/crucible/regions/296a0247-a1f1-4435-8d1c-2c97083c4080","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.023964833Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"datafile"}
{"msg":"snapshot is oxp_489724fa-785f-4f35-bb0f-34adf6a09482/crucible/regions/296a0247-a1f1-4435-8d1c-2c97083c4080@ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.031835017Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_489724fa-785f-4f35-bb0f-34adf6a09482/crucible/regions/296a0247-a1f1-4435-8d1c-2c97083c4080\", \"ed7e43c3-4205-4e31-b98f-42e7a917cf19\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.031866077Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 296a0247-a1f1-4435-8d1c-2c97083c4080-ed7e43c3-4205-4e31-b98f-42e7a917cf19 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.040812744Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"datafile"}
{"msg":"applying SMF actions for region 296a0247-a1f1-4435-8d1c-2c97083c4080 running snapshot ed7e43c3-4205-4e31-b98f-42e7a917cf19 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.041528113Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.041556893Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"uri":"/crucible/0/regions/296a0247-a1f1-4435-8d1c-2c97083c4080/snapshots/ed7e43c3-4205-4e31-b98f-42e7a917cf19/run","method":"POST","req_id":"1dd7d2a4-0413-4da9-9d34-3b50548f0d3a","remote_addr":"[fd00:1122:3344:101::a]:37670","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":26605,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-296a0247-a1f1-4435-8d1c-2c97083c4080-ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.053203896Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-296a0247-a1f1-4435-8d1c-2c97083c4080-ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.05797354Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-296a0247-a1f1-4435-8d1c-2c97083c4080-ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.071768855Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/296a0247-a1f1-4435-8d1c-2c97083c4080/.zfs/snapshot/ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.072098079Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.072146929Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.07219069Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.07222847Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.072276321Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.076523982Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.081192578Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"worker"}
{"msg":"region 296a0247-a1f1-4435-8d1c-2c97083c4080 running snapshot ed7e43c3-4205-4e31-b98f-42e7a917cf19 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.081208728Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"datafile"}
{"msg":"path is \"/data/regions/296a0247-a1f1-4435-8d1c-2c97083c4080\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.282002769Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"datafile"}
{"msg":"dataset is oxp_489724fa-785f-4f35-bb0f-34adf6a09482/crucible/regions/296a0247-a1f1-4435-8d1c-2c97083c4080","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.290816474Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"datafile"}
{"msg":"snapshot is oxp_489724fa-785f-4f35-bb0f-34adf6a09482/crucible/regions/296a0247-a1f1-4435-8d1c-2c97083c4080@ed7e43c3-4205-4e31-b98f-42e7a917cf19","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.299037333Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_489724fa-785f-4f35-bb0f-34adf6a09482/crucible/regions/296a0247-a1f1-4435-8d1c-2c97083c4080\", \"ed7e43c3-4205-4e31-b98f-42e7a917cf19\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.299072873Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-14T22:41:41.307703816Z","hostname":"oxz_crucible_33b9f277-2724-4fde-9002-469a6a2a2f03","pid":26113,"uri":"/crucible/0/regions/296a0247-a1f1-4435-8d1c-2c97083c4080/snapshots/ed7e43c3-4205-4e31-b98f-42e7a917cf19/run","method":"POST","req_id":"adbdea87-52fe-4cd2-8013-6a319501d2bf","remote_addr":"[fd00:1122:3344:101::a]:37670","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":25736,"response_code":"200"}