[ Feb 25 03:48:28 Enabled. ] [ Feb 25 03:48:28 Rereading configuration. ] [ Feb 25 03:48:31 Rereading configuration. ] [ Feb 25 03:48:36 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_6b9444de-c21c-4d5a-9da3-4f3837fdf8f8/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_6b9444de-c21c-4d5a-9da3-4f3837fdf8f8/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:48:36.549047949Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102} {"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:48:36.552178849Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:48:36.552188909Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:48:36.567653525Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:48:36.677690856Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:20.94111496Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:40920"} {"msg":"region 3e817daa-8941-4029-82b0-f521e745de6d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:20.964185066Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:20.967174333Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"uri":"/crucible/0/regions","method":"POST","req_id":"70ec7707-a0ad-4597-bb44-5ec0aa9007e8","remote_addr":"[fd00:1122:3344:101::b]:40920","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":11315,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:20.967198903Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_6b9444de-c21c-4d5a-9da3-4f3837fdf8f8/crucible/regions/3e817daa-8941-4029-82b0-f521e745de6d","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:20.977550605Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_6b9444de-c21c-4d5a-9da3-4f3837fdf8f8/crucible/regions/3e817daa-8941-4029-82b0-f521e745de6d","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:20.977584055Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"3e817daa-8941-4029-82b0-f521e745de6d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3e817daa-8941-4029-82b0-f521e745de6d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.034692973Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"region":"3e817daa-8941-4029-82b0-f521e745de6d","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3e817daa-8941-4029-82b0-f521e745de6d\" \"--data\" \"/data/regions/3e817daa-8941-4029-82b0-f521e745de6d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.036627676Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"region":"3e817daa-8941-4029-82b0-f521e745de6d","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.152032467Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"region":"3e817daa-8941-4029-82b0-f521e745de6d","component":"worker"} {"msg":"region 3e817daa-8941-4029-82b0-f521e745de6d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.152089708Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.1521615Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-3e817daa-8941-4029-82b0-f521e745de6d","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.15261032Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3e817daa-8941-4029-82b0-f521e745de6d","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.156862685Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.167845991Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3e817daa-8941-4029-82b0-f521e745de6d","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.17136738Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3e817daa-8941-4029-82b0-f521e745de6d","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.172116756Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.174298505Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.174350406Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.174408448Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.184559535Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.189502545Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.222619895Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"uri":"/crucible/0/regions","method":"POST","req_id":"b82a3a20-efab-4510-95cf-43cf19c370fd","remote_addr":"[fd00:1122:3344:101::b]:40920","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":59,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.853134726Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"uri":"/crucible/0/regions/3e817daa-8941-4029-82b0-f521e745de6d","method":"GET","req_id":"a85e4497-a48d-4593-9815-822bfe25fb6b","remote_addr":"[fd00:1122:3344:101::b]:40920","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":56,"response_code":"200"} {"msg":"path is \"/data/regions/3e817daa-8941-4029-82b0-f521e745de6d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.856350448Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"datafile"} {"msg":"dataset is oxp_6b9444de-c21c-4d5a-9da3-4f3837fdf8f8/crucible/regions/3e817daa-8941-4029-82b0-f521e745de6d","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.86714938Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"datafile"} {"msg":"region 3e817daa-8941-4029-82b0-f521e745de6d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.878231417Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.87836437Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"disabling downstairs instance: downstairs-3e817daa-8941-4029-82b0-f521e745de6d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.879427014Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.880284963Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"uri":"/crucible/0/regions/3e817daa-8941-4029-82b0-f521e745de6d","method":"DELETE","req_id":"a5c33ba7-0807-453a-a8d7-e8388a9d90f7","remote_addr":"[fd00:1122:3344:101::b]:40920","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":23955,"response_code":"204"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.883915634Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"uri":"/crucible/0/regions/3e817daa-8941-4029-82b0-f521e745de6d","method":"GET","req_id":"a295fd5e-2edb-46f9-892c-065d5fafce98","remote_addr":"[fd00:1122:3344:101::b]:40920","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":35,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.884903297Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_6b9444de-c21c-4d5a-9da3-4f3837fdf8f8/crucible/regions/3e817daa-8941-4029-82b0-f521e745de6d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.896298181Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"region":"3e817daa-8941-4029-82b0-f521e745de6d","component":"worker"} {"msg":"region 3e817daa-8941-4029-82b0-f521e745de6d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:21.960807374Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:49:22.009581785Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"uri":"/crucible/0/regions/3e817daa-8941-4029-82b0-f521e745de6d","method":"GET","req_id":"cf2773cc-bece-47d3-9a26-2a345b67a68f","remote_addr":"[fd00:1122:3344:101::b]:40920","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":70,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:35.75502625Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:38782"} {"msg":"region f6b3e5cd-4e1e-412c-94a9-a195850bb9e8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:35.755147213Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:35.759270735Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:35.759295995Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"uri":"/crucible/0/regions","method":"POST","req_id":"94ca8102-1f35-475e-9c80-bf9eb4995470","remote_addr":"[fd00:1122:3344:101::b]:38782","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":4181,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_6b9444de-c21c-4d5a-9da3-4f3837fdf8f8/crucible/regions/f6b3e5cd-4e1e-412c-94a9-a195850bb9e8","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:35.769271209Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_6b9444de-c21c-4d5a-9da3-4f3837fdf8f8/crucible/regions/f6b3e5cd-4e1e-412c-94a9-a195850bb9e8","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:35.769311449Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"f6b3e5cd-4e1e-412c-94a9-a195850bb9e8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/f6b3e5cd-4e1e-412c-94a9-a195850bb9e8\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:35.847474698Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"region":"f6b3e5cd-4e1e-412c-94a9-a195850bb9e8","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"f6b3e5cd-4e1e-412c-94a9-a195850bb9e8\" \"--data\" \"/data/regions/f6b3e5cd-4e1e-412c-94a9-a195850bb9e8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:35.849239607Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"region":"f6b3e5cd-4e1e-412c-94a9-a195850bb9e8","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.030444739Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"uri":"/crucible/0/regions","method":"POST","req_id":"6feb4406-52d3-4849-8ae3-93ef92a31f36","remote_addr":"[fd00:1122:3344:101::b]:38782","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":908,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.193339083Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"region":"f6b3e5cd-4e1e-412c-94a9-a195850bb9e8","component":"worker"} {"msg":"region f6b3e5cd-4e1e-412c-94a9-a195850bb9e8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.193375183Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.193478906Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-f6b3e5cd-4e1e-412c-94a9-a195850bb9e8","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.194844206Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-f6b3e5cd-4e1e-412c-94a9-a195850bb9e8","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.200083864Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.210455825Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-f6b3e5cd-4e1e-412c-94a9-a195850bb9e8","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.213609575Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/f6b3e5cd-4e1e-412c-94a9-a195850bb9e8","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.21384056Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.213895092Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.213944713Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.213994454Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.217632505Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.222008013Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-02-25T03:59:36.288439949Z","hostname":"oxz_crucible_fb6c0092-7db8-403d-b073-f7fe47700216","pid":23102,"uri":"/crucible/0/regions","method":"POST","req_id":"a131f813-e49c-4f27-a514-cfd8478285dd","remote_addr":"[fd00:1122:3344:101::b]:38782","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":47,"response_code":"200"}