2 * Copyright (c) 2011, 2012, 2013, 2014, 2016 Spectra Logic Corporation
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions, and the following disclaimer,
10 * without modification.
11 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
12 * substantially similar to the "NO WARRANTY" disclaimer below
13 * ("Disclaimer") and any redistribution must be conditioned upon
14 * including a substantially similar Disclaimer requirement for further
15 * binary redistribution.
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
27 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGES.
30 * Authors: Justin T. Gibbs (Spectra Logic Corporation)
36 * We keep case files for any leaf vdev that is not in the optimal state.
37 * However, we only serialize to disk those events that need to be preserved
38 * across reboots. For now, this is just a log of soft errors which we
39 * accumulate in order to mark a device as degraded.
41 #include <sys/cdefs.h>
42 #include <sys/byteorder.h>
45 #include <sys/fs/zfs.h>
62 #include <devdctl/guid.h>
63 #include <devdctl/event.h>
64 #include <devdctl/event_factory.h>
65 #include <devdctl/exception.h>
66 #include <devdctl/consumer.h>
69 #include "vdev_iterator.h"
70 #include "zfsd_event.h"
71 #include "case_file.h"
74 #include "zfsd_exception.h"
75 #include "zpool_list.h"
77 __FBSDID("$FreeBSD$");
79 /*============================ Namespace Control =============================*/
82 using std::stringstream;
87 using DevdCtl::EventFactory;
88 using DevdCtl::EventList;
90 using DevdCtl::ParseException;
92 /*--------------------------------- CaseFile ---------------------------------*/
93 //- CaseFile Static Data -------------------------------------------------------
95 CaseFileList CaseFile::s_activeCases;
96 const string CaseFile::s_caseFilePath = "/var/db/zfsd/cases";
97 const timeval CaseFile::s_removeGracePeriod = { 60 /*sec*/, 0 /*usec*/};
99 //- CaseFile Static Public Methods ---------------------------------------------
101 CaseFile::Find(Guid poolGUID, Guid vdevGUID)
103 for (CaseFileList::iterator curCase = s_activeCases.begin();
104 curCase != s_activeCases.end(); curCase++) {
106 if (((*curCase)->PoolGUID() != poolGUID
107 && Guid::InvalidGuid() != poolGUID)
108 || (*curCase)->VdevGUID() != vdevGUID)
112 * We only carry one active case per-vdev.
120 CaseFile::Find(const string &physPath)
122 CaseFile *result = NULL;
124 for (CaseFileList::iterator curCase = s_activeCases.begin();
125 curCase != s_activeCases.end(); curCase++) {
127 if ((*curCase)->PhysicalPath() != physPath)
130 if (result != NULL) {
131 syslog(LOG_WARNING, "Multiple casefiles found for "
133 "This is most likely a bug in zfsd",
143 CaseFile::ReEvaluateByGuid(Guid poolGUID, const ZfsEvent &event)
145 CaseFileList::iterator casefile;
146 for (casefile = s_activeCases.begin(); casefile != s_activeCases.end();){
147 CaseFileList::iterator next = casefile;
149 if (poolGUID == (*casefile)->PoolGUID())
150 (*casefile)->ReEvaluate(event);
156 CaseFile::Create(Vdev &vdev)
158 CaseFile *activeCase;
160 activeCase = Find(vdev.PoolGUID(), vdev.GUID());
161 if (activeCase == NULL)
162 activeCase = new CaseFile(vdev);
164 return (*activeCase);
168 CaseFile::DeSerialize()
170 struct dirent **caseFiles;
172 int numCaseFiles(scandir(s_caseFilePath.c_str(), &caseFiles,
173 DeSerializeSelector, /*compar*/NULL));
175 if (numCaseFiles == -1)
177 if (numCaseFiles == 0) {
182 for (int i = 0; i < numCaseFiles; i++) {
184 DeSerializeFile(caseFiles[i]->d_name);
193 return (s_activeCases.empty());
199 for (CaseFileList::iterator curCase = s_activeCases.begin();
200 curCase != s_activeCases.end(); curCase++)
208 * Serialize casefiles before deleting them so that they can be reread
209 * and revalidated during BuildCaseFiles.
210 * CaseFiles remove themselves from this list on destruction.
212 while (s_activeCases.size() != 0) {
213 CaseFile *casefile = s_activeCases.front();
214 casefile->Serialize();
220 //- CaseFile Public Methods ----------------------------------------------------
222 CaseFile::RefreshVdevState()
224 ZpoolList zpl(ZpoolList::ZpoolByGUID, &m_poolGUID);
225 zpool_handle_t *casePool(zpl.empty() ? NULL : zpl.front());
226 if (casePool == NULL)
229 Vdev vd(casePool, CaseVdev(casePool));
230 if (vd.DoesNotExist())
233 m_vdevState = vd.State();
234 m_vdevPhysPath = vd.PhysicalPath();
239 CaseFile::ReEvaluate(const string &devPath, const string &physPath, Vdev *vdev)
241 ZpoolList zpl(ZpoolList::ZpoolByGUID, &m_poolGUID);
242 zpool_handle_t *pool(zpl.empty() ? NULL : zpl.front());
244 if (pool == NULL || !RefreshVdevState()) {
246 * The pool or vdev for this case file is no longer
247 * part of the configuration. This can happen
248 * if we process a device arrival notification
249 * before seeing the ZFS configuration change
253 "CaseFile::ReEvaluate(%s,%s) Pool/Vdev unconfigured. "
255 PoolGUIDString().c_str(),
256 VdevGUIDString().c_str());
260 * Since this event was not used to close this
261 * case, do not report it as consumed.
263 return (/*consumed*/false);
266 if (VdevState() > VDEV_STATE_CANT_OPEN) {
268 * For now, newly discovered devices only help for
269 * devices that are missing. In the future, we might
270 * use a newly inserted spare to replace a degraded
273 syslog(LOG_INFO, "CaseFile::ReEvaluate(%s,%s): Pool/Vdev ignored",
274 PoolGUIDString().c_str(), VdevGUIDString().c_str());
275 return (/*consumed*/false);
279 && ( vdev->PoolGUID() == m_poolGUID
280 || vdev->PoolGUID() == Guid::InvalidGuid())
281 && vdev->GUID() == m_vdevGUID) {
283 zpool_vdev_online(pool, vdev->GUIDString().c_str(),
284 ZFS_ONLINE_CHECKREMOVE | ZFS_ONLINE_UNSPARE,
286 syslog(LOG_INFO, "Onlined vdev(%s/%s:%s). State now %s.\n",
287 zpool_get_name(pool), vdev->GUIDString().c_str(),
289 zpool_state_to_name(VdevState(), VDEV_AUX_NONE));
292 * Check the vdev state post the online action to see
293 * if we can retire this case.
297 return (/*consumed*/true);
301 * If the auto-replace policy is enabled, and we have physical
302 * path information, try a physical path replacement.
304 if (zpool_get_prop_int(pool, ZPOOL_PROP_AUTOREPLACE, NULL) == 0) {
306 "CaseFile(%s:%s:%s): AutoReplace not set. "
307 "Ignoring device insertion.\n",
308 PoolGUIDString().c_str(),
309 VdevGUIDString().c_str(),
310 zpool_state_to_name(VdevState(), VDEV_AUX_NONE));
311 return (/*consumed*/false);
314 if (PhysicalPath().empty()) {
316 "CaseFile(%s:%s:%s): No physical path information. "
317 "Ignoring device insertion.\n",
318 PoolGUIDString().c_str(),
319 VdevGUIDString().c_str(),
320 zpool_state_to_name(VdevState(), VDEV_AUX_NONE));
321 return (/*consumed*/false);
324 if (physPath != PhysicalPath()) {
326 "CaseFile(%s:%s:%s): Physical path mismatch. "
327 "Ignoring device insertion.\n",
328 PoolGUIDString().c_str(),
329 VdevGUIDString().c_str(),
330 zpool_state_to_name(VdevState(), VDEV_AUX_NONE));
331 return (/*consumed*/false);
334 /* Write a label on the newly inserted disk. */
335 if (zpool_label_disk(g_zfsHandle, pool, devPath.c_str()) != 0) {
337 "Replace vdev(%s/%s) by physical path (label): %s: %s\n",
338 zpool_get_name(pool), VdevGUIDString().c_str(),
339 libzfs_error_action(g_zfsHandle),
340 libzfs_error_description(g_zfsHandle));
341 return (/*consumed*/false);
344 syslog(LOG_INFO, "CaseFile::ReEvaluate(%s/%s): Replacing with %s",
345 PoolGUIDString().c_str(), VdevGUIDString().c_str(),
347 return (Replace(VDEV_TYPE_DISK, devPath.c_str(), /*isspare*/false));
351 CaseFile::ReEvaluate(const ZfsEvent &event)
353 bool consumed(false);
355 if (event.Value("type") == "misc.fs.zfs.vdev_remove") {
357 * The Vdev we represent has been removed from the
358 * configuration. This case is no longer of value.
362 return (/*consumed*/true);
363 } else if (event.Value("type") == "misc.fs.zfs.pool_destroy") {
364 /* This Pool has been destroyed. Discard the case */
367 return (/*consumed*/true);
368 } else if (event.Value("type") == "misc.fs.zfs.config_sync") {
370 if (VdevState() < VDEV_STATE_HEALTHY)
371 consumed = ActivateSpare();
375 if (event.Value("class") == "resource.fs.zfs.removed") {
376 bool spare_activated;
378 if (!RefreshVdevState()) {
380 * The pool or vdev for this case file is no longer
381 * part of the configuration. This can happen
382 * if we process a device arrival notification
383 * before seeing the ZFS configuration change
387 "CaseFile::ReEvaluate(%s,%s) Pool/Vdev "
388 "unconfigured. Closing\n",
389 PoolGUIDString().c_str(),
390 VdevGUIDString().c_str());
392 * Close the case now so we won't waste cycles in the
398 * Since this event was not used to close this
399 * case, do not report it as consumed.
401 return (/*consumed*/false);
405 * Discard any tentative I/O error events for
406 * this case. They were most likely caused by the
407 * hot-unplug of this device.
409 PurgeTentativeEvents();
411 /* Try to activate spares if they are available */
412 spare_activated = ActivateSpare();
415 * Rescan the drives in the system to see if a recent
416 * drive arrival can be used to solve this case.
418 ZfsDaemon::RequestSystemRescan();
421 * Consume the event if we successfully activated a spare.
422 * Otherwise, leave it in the unconsumed events list so that the
423 * future addition of a spare to this pool might be able to
426 consumed = spare_activated;
427 } else if (event.Value("class") == "resource.fs.zfs.statechange") {
430 * If this vdev is DEGRADED, FAULTED, or UNAVAIL, try to
431 * activate a hotspare. Otherwise, ignore the event
433 if (VdevState() == VDEV_STATE_FAULTED ||
434 VdevState() == VDEV_STATE_DEGRADED ||
435 VdevState() == VDEV_STATE_CANT_OPEN)
436 (void) ActivateSpare();
439 else if (event.Value("class") == "ereport.fs.zfs.io" ||
440 event.Value("class") == "ereport.fs.zfs.checksum") {
442 m_tentativeEvents.push_front(event.DeepCopy());
443 RegisterCallout(event);
447 bool closed(CloseIfSolved());
449 return (consumed || closed);
452 /* Find a Vdev containing the vdev with the given GUID */
454 find_parent(nvlist_t *pool_config, nvlist_t *config, DevdCtl::Guid child_guid)
456 nvlist_t **vdevChildren;
458 unsigned ch, numChildren;
460 error = nvlist_lookup_nvlist_array(config, ZPOOL_CONFIG_CHILDREN,
461 &vdevChildren, &numChildren);
463 if (error != 0 || numChildren == 0)
466 for (ch = 0; ch < numChildren; ch++) {
468 Vdev vdev(pool_config, vdevChildren[ch]);
470 if (vdev.GUID() == child_guid)
473 result = find_parent(pool_config, vdevChildren[ch], child_guid);
482 CaseFile::ActivateSpare() {
483 nvlist_t *config, *nvroot, *parent_config;
485 char *devPath, *vdev_type;
486 const char *poolname;
490 ZpoolList zpl(ZpoolList::ZpoolByGUID, &m_poolGUID);
491 zpool_handle_t *zhp(zpl.empty() ? NULL : zpl.front());
493 syslog(LOG_ERR, "CaseFile::ActivateSpare: Could not find pool "
494 "for pool_guid %" PRIu64".", (uint64_t)m_poolGUID);
497 poolname = zpool_get_name(zhp);
498 config = zpool_get_config(zhp, NULL);
499 if (config == NULL) {
500 syslog(LOG_ERR, "CaseFile::ActivateSpare: Could not find pool "
501 "config for pool %s", poolname);
504 error = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvroot);
506 syslog(LOG_ERR, "CaseFile::ActivateSpare: Could not find vdev "
507 "tree for pool %s", poolname);
511 parent_config = find_parent(config, nvroot, m_vdevGUID);
512 if (parent_config != NULL) {
516 * Don't activate spares for members of a "replacing" vdev.
517 * They're already dealt with. Sparing them will just drag out
518 * the resilver process.
520 error = nvlist_lookup_string(parent_config,
521 ZPOOL_CONFIG_TYPE, &parent_type);
522 if (error == 0 && strcmp(parent_type, VDEV_TYPE_REPLACING) == 0)
527 nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares,
530 /* The pool has no spares configured */
531 syslog(LOG_INFO, "CaseFile::ActivateSpare: "
532 "No spares available for pool %s", poolname);
535 for (i = 0; i < nspares; i++) {
536 uint64_t *nvlist_array;
540 if (nvlist_lookup_uint64_array(spares[i],
541 ZPOOL_CONFIG_VDEV_STATS, &nvlist_array, &nstats) != 0) {
542 syslog(LOG_ERR, "CaseFile::ActivateSpare: Could not "
543 "find vdev stats for pool %s, spare %d",
547 vs = reinterpret_cast<vdev_stat_t *>(nvlist_array);
549 if ((vs->vs_aux != VDEV_AUX_SPARED)
550 && (vs->vs_state == VDEV_STATE_HEALTHY)) {
551 /* We found a usable spare */
557 /* No available spares were found */
561 error = nvlist_lookup_string(spares[i], ZPOOL_CONFIG_PATH, &devPath);
563 syslog(LOG_ERR, "CaseFile::ActivateSpare: Cannot determine "
564 "the path of pool %s, spare %d. Error %d",
569 error = nvlist_lookup_string(spares[i], ZPOOL_CONFIG_TYPE, &vdev_type);
571 syslog(LOG_ERR, "CaseFile::ActivateSpare: Cannot determine "
572 "the vdev type of pool %s, spare %d. Error %d",
577 return (Replace(vdev_type, devPath, /*isspare*/true));
581 CaseFile::RegisterCallout(const Event &event)
583 timeval now, countdown, elapsed, timestamp, zero, remaining;
585 gettimeofday(&now, 0);
586 timestamp = event.GetTimestamp();
587 timersub(&now, ×tamp, &elapsed);
588 timersub(&s_removeGracePeriod, &elapsed, &countdown);
590 * If countdown is <= zero, Reset the timer to the
591 * smallest positive time value instead
594 if (timercmp(&countdown, &zero, <=)) {
595 timerclear(&countdown);
596 countdown.tv_usec = 1;
599 remaining = m_tentativeTimer.TimeRemaining();
601 if (!m_tentativeTimer.IsPending()
602 || timercmp(&countdown, &remaining, <))
603 m_tentativeTimer.Reset(countdown, OnGracePeriodEnded, this);
608 CaseFile::CloseIfSolved()
611 && m_tentativeEvents.empty()) {
614 * We currently do not track or take actions on
615 * devices in the degraded or faulted state.
616 * Once we have support for spare pools, we'll
617 * retain these cases so that any spares added in
618 * the future can be applied to them.
620 switch (VdevState()) {
621 case VDEV_STATE_HEALTHY:
622 /* No need to keep cases for healthy vdevs */
625 case VDEV_STATE_REMOVED:
626 case VDEV_STATE_CANT_OPEN:
628 * Keep open. We may solve it with a newly inserted
631 case VDEV_STATE_FAULTED:
632 case VDEV_STATE_DEGRADED:
634 * Keep open. We may solve it with the future
635 * addition of a spare to the pool
637 case VDEV_STATE_UNKNOWN:
638 case VDEV_STATE_CLOSED:
639 case VDEV_STATE_OFFLINE:
641 * Keep open? This may not be the correct behavior,
642 * but it's what we've always done
648 * Re-serialize the case in order to remove any
649 * previous event data.
660 syslog(LOG_INFO, "CaseFile(%s,%s,%s)\n", PoolGUIDString().c_str(),
661 VdevGUIDString().c_str(), PhysicalPath().c_str());
662 syslog(LOG_INFO, "\tVdev State = %s\n",
663 zpool_state_to_name(VdevState(), VDEV_AUX_NONE));
664 if (m_tentativeEvents.size() != 0) {
665 syslog(LOG_INFO, "\t=== Tentative Events ===\n");
666 for (EventList::iterator event(m_tentativeEvents.begin());
667 event != m_tentativeEvents.end(); event++)
668 (*event)->Log(LOG_INFO);
670 if (m_events.size() != 0) {
671 syslog(LOG_INFO, "\t=== Events ===\n");
672 for (EventList::iterator event(m_events.begin());
673 event != m_events.end(); event++)
674 (*event)->Log(LOG_INFO);
678 //- CaseFile Static Protected Methods ------------------------------------------
680 CaseFile::OnGracePeriodEnded(void *arg)
682 CaseFile &casefile(*static_cast<CaseFile *>(arg));
684 casefile.OnGracePeriodEnded();
688 CaseFile::DeSerializeSelector(const struct dirent *dirEntry)
693 if (dirEntry->d_type == DT_REG
694 && sscanf(dirEntry->d_name, "pool_%" PRIu64 "_vdev_%" PRIu64 ".case",
695 &poolGUID, &vdevGUID) == 2)
701 CaseFile::DeSerializeFile(const char *fileName)
703 string fullName(s_caseFilePath + '/' + fileName);
704 CaseFile *existingCaseFile(NULL);
705 CaseFile *caseFile(NULL);
712 if (sscanf(fileName, "pool_%" PRIu64 "_vdev_%" PRIu64 ".case",
713 &poolGUID, &vdevGUID) != 2) {
714 throw ZfsdException("CaseFile::DeSerialize: "
715 "Unintelligible CaseFile filename %s.\n", fileName);
717 existingCaseFile = Find(Guid(poolGUID), Guid(vdevGUID));
718 if (existingCaseFile != NULL) {
720 * If the vdev is already degraded or faulted,
721 * there's no point in keeping the state around
722 * that we use to put a drive into the degraded
723 * state. However, if the vdev is simply missing,
724 * preserve the case data in the hopes that it will
727 caseFile = existingCaseFile;
728 vdev_state curState(caseFile->VdevState());
729 if (curState > VDEV_STATE_CANT_OPEN
730 && curState < VDEV_STATE_HEALTHY) {
735 ZpoolList zpl(ZpoolList::ZpoolByGUID, &poolGUID);
737 || (vdevConf = VdevIterator(zpl.front())
738 .Find(vdevGUID)) == NULL) {
740 * Either the pool no longer exists
741 * or this vdev is no longer a member of
744 unlink(fullName.c_str());
749 * Any vdev we find that does not have a case file
750 * must be in the healthy state and thus worthy of
751 * continued SERD data tracking.
753 caseFile = new CaseFile(Vdev(zpl.front(), vdevConf));
756 ifstream caseStream(fullName.c_str());
758 throw ZfsdException("CaseFile::DeSerialize: Unable to "
759 "read %s.\n", fileName);
761 caseFile->DeSerialize(caseStream);
762 } catch (const ParseException &exp) {
765 if (caseFile != existingCaseFile)
769 * Since we can't parse the file, unlink it so we don't
770 * trip over it again.
773 } catch (const ZfsdException &zfsException) {
776 if (caseFile != existingCaseFile)
781 //- CaseFile Protected Methods -------------------------------------------------
782 CaseFile::CaseFile(const Vdev &vdev)
783 : m_poolGUID(vdev.PoolGUID()),
784 m_vdevGUID(vdev.GUID()),
785 m_vdevState(vdev.State()),
786 m_vdevPhysPath(vdev.PhysicalPath())
788 stringstream guidString;
790 guidString << m_vdevGUID;
791 m_vdevGUIDString = guidString.str();
793 guidString << m_poolGUID;
794 m_poolGUIDString = guidString.str();
796 s_activeCases.push_back(this);
798 syslog(LOG_INFO, "Creating new CaseFile:\n");
802 CaseFile::~CaseFile()
805 PurgeTentativeEvents();
806 m_tentativeTimer.Stop();
807 s_activeCases.remove(this);
811 CaseFile::PurgeEvents()
813 for (EventList::iterator event(m_events.begin());
814 event != m_events.end(); event++)
821 CaseFile::PurgeTentativeEvents()
823 for (EventList::iterator event(m_tentativeEvents.begin());
824 event != m_tentativeEvents.end(); event++)
827 m_tentativeEvents.clear();
831 CaseFile::SerializeEvList(const EventList events, int fd,
832 const char* prefix) const
836 for (EventList::const_iterator curEvent = events.begin();
837 curEvent != events.end(); curEvent++) {
838 const string &eventString((*curEvent)->GetEventString());
840 // TODO: replace many write(2) calls with a single writev(2)
842 write(fd, prefix, strlen(prefix));
843 write(fd, eventString.c_str(), eventString.length());
848 CaseFile::Serialize()
850 stringstream saveFile;
852 saveFile << setfill('0')
853 << s_caseFilePath << "/"
854 << "pool_" << PoolGUIDString()
855 << "_vdev_" << VdevGUIDString()
858 if (m_events.empty() && m_tentativeEvents.empty()) {
859 unlink(saveFile.str().c_str());
863 int fd(open(saveFile.str().c_str(), O_CREAT|O_TRUNC|O_WRONLY, 0644));
865 syslog(LOG_ERR, "CaseFile::Serialize: Unable to open %s.\n",
866 saveFile.str().c_str());
869 SerializeEvList(m_events, fd);
870 SerializeEvList(m_tentativeEvents, fd, "tentative ");
875 * XXX: This method assumes that events may not contain embedded newlines. If
876 * ever events can contain embedded newlines, then CaseFile must switch
877 * serialization formats
880 CaseFile::DeSerialize(ifstream &caseStream)
883 const EventFactory &factory(ZfsDaemon::Get().GetFactory());
885 caseStream >> std::noskipws >> std::ws;
886 while (caseStream.good()) {
889 * read the beginning of a line and check it for
890 * "tentative". If found, discard "tentative".
894 EventList* destEvents;
895 const string tentFlag("tentative ");
897 std::stringbuf lineBuf;
899 caseStream.get(lineBuf);
900 caseStream.ignore(); /*discard the newline character*/
901 line = lineBuf.str();
902 if (line.compare(0, tentFlag.size(), tentFlag) == 0) {
903 /* Discard "tentative" */
904 line.erase(0, tentFlag.size());
905 destEvents = &m_tentativeEvents;
907 destEvents = &m_events;
909 Event *event(Event::CreateEvent(factory, line));
911 destEvents->push_back(event);
912 RegisterCallout(*event);
921 * This case is no longer relevant. Clean up our
922 * serialization file, and delete the case.
924 syslog(LOG_INFO, "CaseFile(%s,%s) closed - State %s\n",
925 PoolGUIDString().c_str(), VdevGUIDString().c_str(),
926 zpool_state_to_name(VdevState(), VDEV_AUX_NONE));
929 * Serialization of a Case with no event data, clears the
930 * Serialization data for that event.
939 CaseFile::OnGracePeriodEnded()
941 bool should_fault, should_degrade;
942 ZpoolList zpl(ZpoolList::ZpoolByGUID, &m_poolGUID);
943 zpool_handle_t *zhp(zpl.empty() ? NULL : zpl.front());
945 m_events.splice(m_events.begin(), m_tentativeEvents);
946 should_fault = ShouldFault();
947 should_degrade = ShouldDegrade();
949 if (should_fault || should_degrade) {
951 || (VdevIterator(zhp).Find(m_vdevGUID)) == NULL) {
953 * Either the pool no longer exists
954 * or this vdev is no longer a member of
963 /* A fault condition has priority over a degrade condition */
965 /* Fault the vdev and close the case. */
966 if (zpool_vdev_fault(zhp, (uint64_t)m_vdevGUID,
967 VDEV_AUX_ERR_EXCEEDED) == 0) {
968 syslog(LOG_INFO, "Faulting vdev(%s/%s)",
969 PoolGUIDString().c_str(),
970 VdevGUIDString().c_str());
975 syslog(LOG_ERR, "Fault vdev(%s/%s): %s: %s\n",
976 PoolGUIDString().c_str(),
977 VdevGUIDString().c_str(),
978 libzfs_error_action(g_zfsHandle),
979 libzfs_error_description(g_zfsHandle));
982 else if (ShouldDegrade()) {
983 /* Degrade the vdev and close the case. */
984 if (zpool_vdev_degrade(zhp, (uint64_t)m_vdevGUID,
985 VDEV_AUX_ERR_EXCEEDED) == 0) {
986 syslog(LOG_INFO, "Degrading vdev(%s/%s)",
987 PoolGUIDString().c_str(),
988 VdevGUIDString().c_str());
993 syslog(LOG_ERR, "Degrade vdev(%s/%s): %s: %s\n",
994 PoolGUIDString().c_str(),
995 VdevGUIDString().c_str(),
996 libzfs_error_action(g_zfsHandle),
997 libzfs_error_description(g_zfsHandle));
1004 CaseFile::BeingReplacedBy(zpool_handle_t *zhp) {
1005 Vdev vd(zhp, CaseVdev(zhp));
1006 std::list<Vdev> children;
1007 std::list<Vdev>::iterator children_it;
1009 Vdev parent(vd.Parent());
1010 Vdev replacing(NonexistentVdev);
1013 * To determine whether we are being replaced by another spare that
1014 * is still working, then make sure that it is currently spared and
1015 * that the spare is either resilvering or healthy. If any of these
1016 * conditions fail, then we are not being replaced by a spare.
1018 * If the spare is healthy, then the case file should be closed very
1019 * soon after this check.
1021 if (parent.DoesNotExist()
1022 || parent.Name(zhp, /*verbose*/false) != "spare")
1023 return (NonexistentVdev);
1025 children = parent.Children();
1026 children_it = children.begin();
1027 for (;children_it != children.end(); children_it++) {
1028 Vdev child = *children_it;
1030 /* Skip our vdev. */
1031 if (child.GUID() == VdevGUID())
1034 * Accept the first child that doesn't match our GUID, or
1035 * any resilvering/healthy device if one exists.
1037 if (replacing.DoesNotExist() || child.IsResilvering()
1038 || child.State() == VDEV_STATE_HEALTHY)
1046 CaseFile::Replace(const char* vdev_type, const char* path, bool isspare) {
1047 nvlist_t *nvroot, *newvd;
1048 const char *poolname;
1049 string oldstr(VdevGUIDString());
1052 /* Figure out what pool we're working on */
1053 ZpoolList zpl(ZpoolList::ZpoolByGUID, &m_poolGUID);
1054 zpool_handle_t *zhp(zpl.empty() ? NULL : zpl.front());
1056 syslog(LOG_ERR, "CaseFile::Replace: could not find pool for "
1057 "pool_guid %" PRIu64 ".", (uint64_t)m_poolGUID);
1060 poolname = zpool_get_name(zhp);
1061 Vdev vd(zhp, CaseVdev(zhp));
1062 Vdev replaced(BeingReplacedBy(zhp));
1064 if (isspare && !vd.IsSpare() && !replaced.DoesNotExist()) {
1065 /* If we are already being replaced by a working spare, pass. */
1066 if (replaced.IsResilvering()
1067 || replaced.State() == VDEV_STATE_HEALTHY) {
1068 syslog(LOG_INFO, "CaseFile::Replace(%s->%s): already "
1069 "replaced", VdevGUIDString().c_str(), path);
1070 return (/*consumed*/false);
1073 * If we have already been replaced by a spare, but that spare
1074 * is broken, we must spare the spare, not the original device.
1076 oldstr = replaced.GUIDString();
1077 syslog(LOG_INFO, "CaseFile::Replace(%s->%s): sparing "
1078 "broken spare %s instead", VdevGUIDString().c_str(),
1079 path, oldstr.c_str());
1083 * Build a root vdev/leaf vdev configuration suitable for
1084 * zpool_vdev_attach. Only enough data for the kernel to find
1085 * the device (i.e. type and disk device node path) are needed.
1090 if (nvlist_alloc(&nvroot, NV_UNIQUE_NAME, 0) != 0
1091 || nvlist_alloc(&newvd, NV_UNIQUE_NAME, 0) != 0) {
1092 syslog(LOG_ERR, "Replace vdev(%s/%s): Unable to allocate "
1093 "configuration data.", poolname, oldstr.c_str());
1095 nvlist_free(nvroot);
1098 if (nvlist_add_string(newvd, ZPOOL_CONFIG_TYPE, vdev_type) != 0
1099 || nvlist_add_string(newvd, ZPOOL_CONFIG_PATH, path) != 0
1100 || nvlist_add_string(nvroot, ZPOOL_CONFIG_TYPE, VDEV_TYPE_ROOT) != 0
1101 || nvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
1103 syslog(LOG_ERR, "Replace vdev(%s/%s): Unable to initialize "
1104 "configuration data.", poolname, oldstr.c_str());
1106 nvlist_free(nvroot);
1110 /* Data was copied when added to the root vdev. */
1113 retval = (zpool_vdev_attach(zhp, oldstr.c_str(), path, nvroot,
1114 /*replace*/B_TRUE, /*rebuild*/ B_FALSE) == 0);
1116 syslog(LOG_INFO, "Replacing vdev(%s/%s) with %s\n",
1117 poolname, oldstr.c_str(), path);
1119 syslog(LOG_ERR, "Replace vdev(%s/%s): %s: %s\n",
1120 poolname, oldstr.c_str(), libzfs_error_action(g_zfsHandle),
1121 libzfs_error_description(g_zfsHandle));
1122 nvlist_free(nvroot);
1127 /* Does the argument event refer to a checksum error? */
1129 IsChecksumEvent(const Event* const event)
1131 return ("ereport.fs.zfs.checksum" == event->Value("type"));
1134 /* Does the argument event refer to an IO error? */
1136 IsIOEvent(const Event* const event)
1138 return ("ereport.fs.zfs.io" == event->Value("type"));
1142 CaseFile::ShouldDegrade() const
1144 return (std::count_if(m_events.begin(), m_events.end(),
1145 IsChecksumEvent) > ZFS_DEGRADE_IO_COUNT);
1149 CaseFile::ShouldFault() const
1151 return (std::count_if(m_events.begin(), m_events.end(),
1152 IsIOEvent) > ZFS_DEGRADE_IO_COUNT);
1156 CaseFile::CaseVdev(zpool_handle_t *zhp) const
1158 return (VdevIterator(zhp).Find(VdevGUID()));