code example

#include <QWidget>

class GreyMaskWidget : public QWidget
{
    Q_OBJECT
public:
    GreyMaskWidget(QWidget *target, QWidget *parent = nullptr);
    ~GreyMaskWidget();

protected:
    virtual void paintEvent(QPaintEvent *event) override;
    virtual bool eventFilter(QObject *obj, QEvent *event) override;

private:
    QWidget *m_target;
};

#include <QPainter>
#include <QEvent>
#include <QPainterPath>
#include <QtMath>

GreyMaskWidget::GreyMaskWidget(QWidget *target, QWidget *parent)
    : QWidget(parent)
    , m_target(target)
{
    setAttribute(Qt::WA_TranslucentBackground);
    setWindowFlags(Qt::FramelessWindowHint | Qt::Window);

    target->installEventFilter(this);
}

GreyMaskWidget::~GreyMaskWidget()
{

}

void GreyMaskWidget::paintEvent(QPaintEvent *event)
{
    if (m_target == nullptr)
        return;

    QPainter painter(this);
    painter.setBrush(Qt::NoBrush);
    QPainterPath path;
    path.setFillRule(Qt::WindingFill);

    QRect targetRect = m_target->frameGeometry();
    QRectF rect(targetRect.topLeft(), QSize(targetRect.width() + 20, targetRect.height() + 20));
    rect.moveTo(rect.topLeft() - QPoint(10, 10));
    path.addRoundedRect(rect, 8, 8);

    painter.setRenderHint(QPainter::Antialiasing, true);

    QColor color(0, 0, 0, 30);
    for (int i = 0; i < 10; i++)
    {
        QPainterPath path;
        path.setFillRule(Qt::WindingFill);
        path.addRect(10 - i, 10 - i, targetRect.width() + i * 2, targetRect.height() + i * 2);
        color.setAlpha(150 - qSqrt(i) * 50);
        painter.setPen(color);
        painter.drawPath(path);
    }
    QWidget::paintEvent(event);
}

bool GreyMaskWidget::eventFilter(QObject *obj, QEvent *event)
{
    if (obj == m_target && event->type() == QEvent::Hide)
    {
        hide();
    }
    else if (obj == m_target && event->type() == QEvent::Show)
    {
        QRect rect = m_target->frameGeometry();
        QPoint topLeft = rect.topLeft();
        rect.moveTo(topLeft - QPoint(10, 10));
        rect.setWidth(rect.width() + 20);
        rect.setHeight(rect.height() + 20);
        setGeometry(rect);
        show();
    }
    else if (obj == m_target && event->type() == QEvent::Move)
    {
        QPoint targetPos = m_target->frameGeometry().topLeft() - QPoint(10, 10);
        move(targetPos);
    }
    else if (obj == m_target && event->type() == QEvent::Resize)
    {
        QPoint targetPos = m_target->frameGeometry().topLeft() - QPoint(10, 10);
        QRect targetRect(targetPos, m_target->size() + QSize(20, 20));
        setGeometry(targetRect);
    }
    return QWidget::eventFilter(obj, event);
}

how to use it.

ExampleDialog::ExampleDialog(QWidget *parent)
    : QDialog(parent)
{
    setWindowFlags(Qt::Window | Qt::FramelessWindowHint);
    initUI();
    new GreyMaskWidget(this, this);  // <---
}