[ Jan 27 09:31:32 Enabled. ]
[ Jan 27 09:31:32 Rereading configuration. ]
[ Jan 27 09:31:33 Rereading configuration. ]
[ Jan 27 09:31:36 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:31:36.831563472Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:31:36.837743146Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:31:36.837790707Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:31:36.845815693Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:31:36.940701218Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.062660818Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:44885"}
{"msg":"region 0e6e0429-91d5-4e22-8918-d53a78ae092d state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.099736659Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.102757209Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions","method":"POST","req_id":"ad43026f-0bb4-4d7f-80be-50c3f22e0ca8","remote_addr":"[fd00:1122:3344:101::b]:44885","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":15754,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.105238987Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/0e6e0429-91d5-4e22-8918-d53a78ae092d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.115186728Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/0e6e0429-91d5-4e22-8918-d53a78ae092d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.115224969Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"0e6e0429-91d5-4e22-8918-d53a78ae092d\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/0e6e0429-91d5-4e22-8918-d53a78ae092d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.209938209Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"region":"0e6e0429-91d5-4e22-8918-d53a78ae092d","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"0e6e0429-91d5-4e22-8918-d53a78ae092d\" \"--data\" \"/data/regions/0e6e0429-91d5-4e22-8918-d53a78ae092d\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.210015551Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"region":"0e6e0429-91d5-4e22-8918-d53a78ae092d","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.394222161Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions","method":"POST","req_id":"f24003d0-5cd3-4703-ad9b-fa6616cc32b2","remote_addr":"[fd00:1122:3344:101::b]:44885","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":94,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.555394555Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"region":"0e6e0429-91d5-4e22-8918-d53a78ae092d","component":"worker"}
{"msg":"region 0e6e0429-91d5-4e22-8918-d53a78ae092d state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.555477287Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.555737403Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-0e6e0429-91d5-4e22-8918-d53a78ae092d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.556190904Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-0e6e0429-91d5-4e22-8918-d53a78ae092d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.561797304Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.589503648Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-0e6e0429-91d5-4e22-8918-d53a78ae092d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.618369488Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/0e6e0429-91d5-4e22-8918-d53a78ae092d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.618834469Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.618917131Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.618955542Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.619001873Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.633970081Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.651513908Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:18.942166481Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions","method":"POST","req_id":"f710495e-093d-48b8-8e72-5cabb09c72e7","remote_addr":"[fd00:1122:3344:101::b]:44885","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:19.447571294Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions/0e6e0429-91d5-4e22-8918-d53a78ae092d","method":"GET","req_id":"c6eb9c66-30be-4163-806a-eac3459b9114","remote_addr":"[fd00:1122:3344:101::b]:44885","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":58,"response_code":"200"}
{"msg":"path is \"/data/regions/0e6e0429-91d5-4e22-8918-d53a78ae092d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:19.450838769Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"dataset is oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/0e6e0429-91d5-4e22-8918-d53a78ae092d","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:19.461944677Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"region 0e6e0429-91d5-4e22-8918-d53a78ae092d state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:19.472206666Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:19.472423161Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:19.472513933Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions/0e6e0429-91d5-4e22-8918-d53a78ae092d","method":"DELETE","req_id":"5a8f9d8b-4f25-4c56-a864-db366ba39c0c","remote_addr":"[fd00:1122:3344:101::b]:44885","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":21714,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-0e6e0429-91d5-4e22-8918-d53a78ae092d (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:19.473493956Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:19.475672786Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions/0e6e0429-91d5-4e22-8918-d53a78ae092d","method":"GET","req_id":"25371a3d-8e76-4dc4-a439-0a96bbe87abe","remote_addr":"[fd00:1122:3344:101::b]:44885","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:19.48273712Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/0e6e0429-91d5-4e22-8918-d53a78ae092d\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:19.494141145Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"region":"0e6e0429-91d5-4e22-8918-d53a78ae092d","component":"worker"}
{"msg":"region 0e6e0429-91d5-4e22-8918-d53a78ae092d state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:19.572437064Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:19.627361891Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions/0e6e0429-91d5-4e22-8918-d53a78ae092d","method":"GET","req_id":"f12ebfd9-1bbd-41ef-8a77-9d612dd7fea4","remote_addr":"[fd00:1122:3344:101::b]:44885","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"region 15902abf-f00e-422e-8de1-b7634529d7ed state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:21.829448752Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:21.829628866Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions","method":"POST","req_id":"62ce3bfa-c5ba-4ab9-b3ac-987af6f18e0e","remote_addr":"[fd00:1122:3344:101::b]:44885","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":243,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:21.829686467Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/15902abf-f00e-422e-8de1-b7634529d7ed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:21.838979694Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/15902abf-f00e-422e-8de1-b7634529d7ed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:21.839008794Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"15902abf-f00e-422e-8de1-b7634529d7ed\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/15902abf-f00e-422e-8de1-b7634529d7ed\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:21.931447382Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"region":"15902abf-f00e-422e-8de1-b7634529d7ed","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"15902abf-f00e-422e-8de1-b7634529d7ed\" \"--data\" \"/data/regions/15902abf-f00e-422e-8de1-b7634529d7ed\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:21.933329735Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"region":"15902abf-f00e-422e-8de1-b7634529d7ed","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.03465742Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions","method":"POST","req_id":"9bf6a449-cbfb-4c68-bdc6-7a4aed6134e7","remote_addr":"[fd00:1122:3344:101::b]:44885","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.109355675Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"region":"15902abf-f00e-422e-8de1-b7634529d7ed","component":"worker"}
{"msg":"region 15902abf-f00e-422e-8de1-b7634529d7ed state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.109383616Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.109520659Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-15902abf-f00e-422e-8de1-b7634529d7ed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.110911161Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-15902abf-f00e-422e-8de1-b7634529d7ed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.124685561Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.140029637Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-15902abf-f00e-422e-8de1-b7634529d7ed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.149469277Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/15902abf-f00e-422e-8de1-b7634529d7ed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.149786524Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.149835215Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.149881837Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.149921887Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.156582362Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.167584238Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:32:22.699183349Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions","method":"POST","req_id":"586312bc-0124-4335-bc4e-8588652194d2","remote_addr":"[fd00:1122:3344:101::b]:44885","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.448793353Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:46911"}
{"msg":"region 28714af9-6f6d-477b-8e29-70d042d52e97 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.449449348Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.45037723Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions","method":"POST","req_id":"f4471534-b41d-4c3b-89f7-cbbf4560b8dd","remote_addr":"[fd00:1122:3344:101::b]:46911","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":1532,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.45039241Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/28714af9-6f6d-477b-8e29-70d042d52e97","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.459920492Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/28714af9-6f6d-477b-8e29-70d042d52e97","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.459958173Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"28714af9-6f6d-477b-8e29-70d042d52e97\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/28714af9-6f6d-477b-8e29-70d042d52e97\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.538880936Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"region":"28714af9-6f6d-477b-8e29-70d042d52e97","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"28714af9-6f6d-477b-8e29-70d042d52e97\" \"--data\" \"/data/regions/28714af9-6f6d-477b-8e29-70d042d52e97\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.540875483Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"region":"28714af9-6f6d-477b-8e29-70d042d52e97","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.644401818Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"region":"28714af9-6f6d-477b-8e29-70d042d52e97","component":"worker"}
{"msg":"region 28714af9-6f6d-477b-8e29-70d042d52e97 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.644458729Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.644607693Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-28714af9-6f6d-477b-8e29-70d042d52e97","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.652388594Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-28714af9-6f6d-477b-8e29-70d042d52e97","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.668493828Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.681735085Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-28714af9-6f6d-477b-8e29-70d042d52e97","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.688671006Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/28714af9-6f6d-477b-8e29-70d042d52e97","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.689076676Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.689143117Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.689207359Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.68926208Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.698246699Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.707639167Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:09.718932089Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions","method":"POST","req_id":"5d8d47bc-963d-45e7-a7c6-999e54173757","remote_addr":"[fd00:1122:3344:101::b]:46911","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.162265922Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions/15902abf-f00e-422e-8de1-b7634529d7ed","method":"GET","req_id":"ebf6968f-09f5-42f3-b86e-d1c7cd417a54","remote_addr":"[fd00:1122:3344:101::b]:46911","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":118,"response_code":"200"}
{"msg":"path is \"/data/regions/15902abf-f00e-422e-8de1-b7634529d7ed\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.166838919Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"dataset is oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/15902abf-f00e-422e-8de1-b7634529d7ed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.179341309Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"snapshot is oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/15902abf-f00e-422e-8de1-b7634529d7ed@a64ec56a-0146-486a-8266-3727e23c9078","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.191538242Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/15902abf-f00e-422e-8de1-b7634529d7ed\", \"a64ec56a-0146-486a-8266-3727e23c9078\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.191595424Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.20436888Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions/15902abf-f00e-422e-8de1-b7634529d7ed/snapshots/a64ec56a-0146-486a-8266-3727e23c9078","method":"GET","req_id":"af76304d-33d7-431e-896a-8dc97ab012a3","remote_addr":"[fd00:1122:3344:101::b]:46911","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":37551,"response_code":"200"}
{"msg":"path is \"/data/regions/15902abf-f00e-422e-8de1-b7634529d7ed\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.209400997Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"dataset is oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/15902abf-f00e-422e-8de1-b7634529d7ed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.221183971Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"snapshot is oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/15902abf-f00e-422e-8de1-b7634529d7ed@a64ec56a-0146-486a-8266-3727e23c9078","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.233338833Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/15902abf-f00e-422e-8de1-b7634529d7ed\", \"a64ec56a-0146-486a-8266-3727e23c9078\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.233394975Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 15902abf-f00e-422e-8de1-b7634529d7ed-a64ec56a-0146-486a-8266-3727e23c9078 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.245648739Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"applying SMF actions for region 15902abf-f00e-422e-8de1-b7634529d7ed running snapshot a64ec56a-0146-486a-8266-3727e23c9078 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.245893835Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.245932506Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions/15902abf-f00e-422e-8de1-b7634529d7ed/snapshots/a64ec56a-0146-486a-8266-3727e23c9078/run","method":"POST","req_id":"c6c3b47f-8c20-4651-bf06-cadb8c198587","remote_addr":"[fd00:1122:3344:101::b]:46911","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":36591,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-15902abf-f00e-422e-8de1-b7634529d7ed-a64ec56a-0146-486a-8266-3727e23c9078","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.249750595Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-15902abf-f00e-422e-8de1-b7634529d7ed-a64ec56a-0146-486a-8266-3727e23c9078","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.25645219Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-15902abf-f00e-422e-8de1-b7634529d7ed-a64ec56a-0146-486a-8266-3727e23c9078","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.274727405Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/15902abf-f00e-422e-8de1-b7634529d7ed/.zfs/snapshot/a64ec56a-0146-486a-8266-3727e23c9078","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.275159395Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.275231847Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.275287748Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.275341429Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.275407991Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.282711981Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.289334305Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"worker"}
{"msg":"region 15902abf-f00e-422e-8de1-b7634529d7ed running snapshot a64ec56a-0146-486a-8266-3727e23c9078 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.289357745Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"path is \"/data/regions/15902abf-f00e-422e-8de1-b7634529d7ed\"","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.494039981Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"dataset is oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/15902abf-f00e-422e-8de1-b7634529d7ed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.502976668Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"datafile"}
{"msg":"snapshot is oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/15902abf-f00e-422e-8de1-b7634529d7ed@a64ec56a-0146-486a-8266-3727e23c9078","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.511286921Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_fffdedc2-8da9-4b18-8b94-676a50c56a55/crucible/regions/15902abf-f00e-422e-8de1-b7634529d7ed\", \"a64ec56a-0146-486a-8266-3727e23c9078\"]","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.511334842Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-01-27T09:43:11.518671123Z","hostname":"oxz_crucible_1a51ed25-62e8-4e11-8d4e-27a2a38a2e41","pid":24634,"uri":"/crucible/0/regions/15902abf-f00e-422e-8de1-b7634529d7ed/snapshots/a64ec56a-0146-486a-8266-3727e23c9078/run","method":"POST","req_id":"4eab7a5d-43c1-4e5d-8938-16cb50e1bfbe","remote_addr":"[fd00:1122:3344:101::b]:46911","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":24684,"response_code":"200"}